mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-11 21:59:38 +01:00
fix: enforce accounting coverage and recovery regression gates
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
c1102f05d4
commit
ec91c9c709
45 files changed
+5400
-20
No files matched your search
@@ -0,0 +1,169 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { reserveRunBudget } from "../src/services/budget-reservations.js";
|
||||
import { createRunUsageRecorder, indexPendingUsageReceipts, usageReceiptSpoolPath } from "../src/services/usage-receipts.js";
|
||||
import { budgetService } from "../src/services/budgets.js";
|
||||
import { sql } from "drizzle-orm";
|
||||
/** Reproducible synthetic PostgreSQL benchmark. Never reads DATABASE_URL or
|
||||
* provider credentials; startEmbeddedPostgresTestDatabase owns all data. */
|
||||
import { performance, monitorEventLoopDelay } from "node:perf_hooks";
|
||||
import { mkdir, writeFile, mkdtemp, rm } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { agents, companies, projects, heartbeatRuns, budgetPolicies, createDb, startEmbeddedPostgresTestDatabase } from "@paperclipai/db";
|
||||
import { costService } from "../src/services/costs.js";
|
||||
import { withAccountingTransaction } from "../src/services/accounting-transaction.js";
|
||||
import { accountingIntegrityService } from "../src/services/accounting-integrity.js";
|
||||
import { reconcileRunCosts } from "../src/services/run-cost-accounting.js";
|
||||
const count = Number(process.env.PAPERCLIP_ACCOUNTING_BENCH_ROWS ?? 1_000_000);
|
||||
if (!Number.isInteger(count) || count < 1000 || count > 10_000_000) throw new Error("Benchmark row count must be 1,000–10,000,000");
|
||||
// Recovery reads the instance receipt spool as well as the database. Always
|
||||
// own both; a caller's real PAPERCLIP_HOME must never be scanned or replayed.
|
||||
const previousHome = process.env.PAPERCLIP_HOME;
|
||||
const benchmarkHome = await mkdtemp(path.join(os.tmpdir(), "accounting-benchmark-"));
|
||||
process.env.PAPERCLIP_HOME = benchmarkHome;
|
||||
const database = await startEmbeddedPostgresTestDatabase("accounting-scale-").catch(async error => {
|
||||
if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome;
|
||||
await rm(benchmarkHome, { recursive: true, force: true });
|
||||
throw error;
|
||||
});
|
||||
const db = createDb(database.connectionString);
|
||||
const started = performance.now();
|
||||
const eventLoop = monitorEventLoopDelay({ resolution: 10 }); eventLoop.enable();
|
||||
function distribution(values: number[]) {
|
||||
const sorted = [...values].sort((a,b) => a-b);
|
||||
const pick = (p: number) => Number(sorted[Math.min(sorted.length - 1, Math.ceil(p * sorted.length) - 1)].toFixed(2));
|
||||
return { samples: sorted.length, p50Ms: pick(0.5), p95Ms: pick(0.95), p99Ms: pick(0.99), maxMs: pick(1) };
|
||||
}
|
||||
async function time(work: () => Promise<unknown>) { const start = performance.now(); await work(); return performance.now() - start; }
|
||||
try {
|
||||
const [company] = await db.insert(companies).values({ name: "Scale fixture", issuePrefix: "SCALE" }).returning();
|
||||
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process", status: "idle" }).returning();
|
||||
const [project] = await db.insert(projects).values({ companyId: company.id, name: "Hot project" }).returning();
|
||||
const seedMs = await time(async () => {
|
||||
await db.$client`insert into cost_events (company_id,agent_id,project_id,provider,biller,model,cost_cents,input_tokens,cached_input_tokens,output_tokens,occurred_at)
|
||||
select ${company.id},${agent.id}, case when i % 100 = 0 then ${project.id}::uuid else null end, 'fixture','fixture','fixture', 0.1234567,7,11,3,
|
||||
date_trunc('month',now() at time zone 'UTC') at time zone 'UTC' - (i % 12) * interval '1 month'
|
||||
from generate_series(1,${count}) i`;
|
||||
await db.$client`analyze cost_events`;
|
||||
// Seed projections through an explicit reviewed repair, not benchmark-only SQL.
|
||||
const integrity = accountingIntegrityService(db); const review = await integrity.inspect(company.id);
|
||||
await integrity.repair(company.id, review.fingerprint, "Initialize synthetic benchmark projections", "benchmark");
|
||||
});
|
||||
const reports: number[] = [], projectReports: number[] = [];
|
||||
for (let i = 0; i < 20; i++) {
|
||||
reports.push(await time(() => costService(db).summary(company.id, { allTime: true })));
|
||||
projectReports.push(await time(() => costService(db).byProject(company.id, { allTime: true })));
|
||||
}
|
||||
const writes: number[] = []; const concurrency = 8; const writeStarted = performance.now();
|
||||
for (let batch = 0; batch < 8; batch++) await Promise.all(Array.from({ length: concurrency }, (_, index) => time(() => costService(db).createEvent(company.id, {
|
||||
agentId: agent.id, provider: "fixture", model: "fixture", costCents: "0.0000001", idempotencyKey: `bench:${batch}:${index}`, occurredAt: new Date(),
|
||||
})).then(ms => writes.push(ms))));
|
||||
const writeMs = performance.now() - writeStarted;
|
||||
await budgetService(db).upsertPolicy(company.id, { scopeType: "company", scopeId: company.id, amount: 2_000_000_000 }, "benchmark");
|
||||
await budgetService(db).upsertPolicy(company.id, { scopeType: "agent", scopeId: agent.id, amount: 2_000_000_000 }, "benchmark");
|
||||
const budgetedWrites: number[] = [];
|
||||
const budgetedStart = performance.now();
|
||||
for (let batch = 0; batch < 4; batch++) await Promise.all(Array.from({ length: concurrency }, (_, index) => time(() => costService(db).createEvent(company.id, {
|
||||
agentId: agent.id, provider: "fixture", model: "fixture", costCents: "0.0000001", idempotencyKey: `budgeted:${batch}:${index}`, occurredAt: new Date(),
|
||||
})).then(ms => budgetedWrites.push(ms))));
|
||||
const budgetedMs = performance.now() - budgetedStart;
|
||||
const admissions: number[] = [];
|
||||
for (let i = 0; i < 20; i++) {
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "running" }).returning();
|
||||
admissions.push(await time(() => reserveRunBudget(db, company.id, run.id, null)));
|
||||
}
|
||||
const dashboard: number[] = [], writesWhilePolling: number[] = [];
|
||||
// Eight viewers each issue the ledger queries used by the Costs overview,
|
||||
// together with budget and accounting health reads, while eight writes run.
|
||||
for (let batch = 0; batch < 4; batch++) await Promise.all([
|
||||
...Array.from({ length: 8 }, () => time(() => Promise.all([
|
||||
costService(db).summary(company.id), costService(db).byAgent(company.id),
|
||||
costService(db).byProject(company.id), costService(db).byAgentModel(company.id),
|
||||
budgetService(db).overview(company.id), accountingIntegrityService(db).health(company.id),
|
||||
])).then(ms => dashboard.push(ms))),
|
||||
...Array.from({ length: concurrency }, (_, index) => time(() => costService(db).createEvent(company.id, {
|
||||
agentId: agent.id, provider: "fixture", model: "fixture", costCents: "0.0000001", idempotencyKey: `polling:${batch}:${index}`, occurredAt: new Date(),
|
||||
})).then(ms => writesWhilePolling.push(ms))),
|
||||
]);
|
||||
const [checkpointRun] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "running" }).returning();
|
||||
const recorder = await createRunUsageRecorder(db, { companyId: company.id, runId: checkpointRun.id, adapterType: "process" });
|
||||
const checkpoints: number[] = [];
|
||||
for (let i = 1; i <= 50; i++) checkpoints.push(await time(() => recorder.capture({ complete: false, usageBasis: "per_run", usage: { inputTokens: i, outputTokens: i }, costUsdExact: "0.000000001" })));
|
||||
// Include historical run-table size, independently of the pending index.
|
||||
await db.$client`insert into heartbeat_runs (company_id,agent_id,status,cost_accounting_pending,finished_at)
|
||||
select ${company.id},${agent.id},'succeeded',false, now() from generate_series(1,100000)`;
|
||||
await db.$client`analyze heartbeat_runs`;
|
||||
const health: number[] = [];
|
||||
for (let i = 0; i < 20; i++) health.push(await time(() => accountingIntegrityService(db).health(company.id)));
|
||||
const waits: number[] = [];
|
||||
await Promise.all(Array.from({ length: concurrency }, () => {
|
||||
const start = performance.now();
|
||||
return withAccountingTransaction(db, company.id, async tx => { waits.push(performance.now() - start); await tx.execute(sql`select pg_sleep(0.005)`); });
|
||||
}));
|
||||
await db.$client`insert into heartbeat_runs (company_id,agent_id,status,cost_accounting_pending,usage_json,finished_at)
|
||||
select ${company.id},${agent.id},'failed',true,'{"accountingReceiptReady":true,"costUsdExact":"0.000000001","inputTokens":1}'::jsonb, now() from generate_series(1,250)`;
|
||||
const recovery: Array<{ milliseconds: number; scanned: number; accounted: number }> = [];
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const before = performance.now(); const result = await reconcileRunCosts(db); recovery.push({ milliseconds: performance.now()-before, ...result });
|
||||
}
|
||||
if (recovery.reduce((sum, r) => sum+r.accounted,0) !== 250) throw new Error("Benchmark recovery did not conserve receipt count");
|
||||
const spool = usageReceiptSpoolPath(); await mkdir(spool, { recursive: true });
|
||||
const spoolBacklogs: Array<{ files: number; indexMs: number; recorderStartMs: number }> = [];
|
||||
const foreignRunId = randomUUID();
|
||||
let published = 0;
|
||||
for (const files of [1000, 10000]) {
|
||||
// Publish synthetic immutable files directly; timed checkpoints above use
|
||||
// the real fsync path. These belong to another run and must be retained.
|
||||
for (; published < files; published += 50) await Promise.all(Array.from({ length: 50 }, (_, offset) => writeFile(path.join(spool, `${published + offset}.json`), JSON.stringify({
|
||||
schema: "paperclip/accounting-receipt/v1", id: randomUUID(), companyId: company.id, runId: foreignRunId,
|
||||
sourceId: randomUUID(), sequence: 1, receivedAt: new Date().toISOString(), adapterType: "process", receipt: { complete: false },
|
||||
}))));
|
||||
const indexMs = await time(() => indexPendingUsageReceipts());
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "running" }).returning();
|
||||
const recorderStartMs = await time(() => createRunUsageRecorder(db, { companyId: company.id, runId: run.id, adapterType: "process" }));
|
||||
spoolBacklogs.push({ files, indexMs, recorderStartMs });
|
||||
}
|
||||
await rm(spool, { recursive: true, force: true });
|
||||
// A large policy list must not make each write fetch every agent's policies.
|
||||
const extraAgents = await db.insert(agents).values(Array.from({ length: 100 }, (_, i) => ({ companyId: company.id, name: `Idle ${i}`, role: "engineer", adapterType: "process" }))).returning();
|
||||
await db.insert(budgetPolicies).values(extraAgents.map(extra => ({ companyId: company.id, scopeType: "agent", scopeId: extra.id, windowKind: "calendar_month_utc", amount: 2_000_000_000 })));
|
||||
const largeOverview: number[] = [];
|
||||
for (let i = 0; i < 10; i++) largeOverview.push(await time(async () => {
|
||||
const overview = await budgetService(db).overview(company.id);
|
||||
if (overview.policies.length !== 102) throw new Error("Benchmark policy fixture is incomplete");
|
||||
}));
|
||||
const largePolicyWrites: number[] = [], largePolicyAdmissions: number[] = [];
|
||||
const largePolicyStart = performance.now();
|
||||
for (let batch = 0; batch < 4; batch++) await Promise.all(Array.from({ length: concurrency }, (_, index) => time(() => costService(db).createEvent(company.id, {
|
||||
agentId: agent.id, provider: "fixture", model: "fixture", costCents: "0.0000001", idempotencyKey: `large-policy:${batch}:${index}`, occurredAt: new Date(),
|
||||
})).then(ms => largePolicyWrites.push(ms))));
|
||||
const largePolicyMs = performance.now() - largePolicyStart;
|
||||
for (let i = 0; i < 20; i++) {
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "running" }).returning();
|
||||
largePolicyAdmissions.push(await time(() => reserveRunBudget(db, company.id, run.id, null)));
|
||||
}
|
||||
const integrity = await accountingIntegrityService(db).inspect(company.id);
|
||||
if (integrity.findings.length) throw new Error(`Benchmark violated accounting integrity: ${JSON.stringify(integrity.findings)}`);
|
||||
const plan = await db.$client`explain (analyze,buffers,format json) select sum(cost_cents) from cost_events where company_id = ${company.id} and project_id = ${project.id}
|
||||
and occurred_at >= date_trunc('month',now() at time zone 'UTC') at time zone 'UTC'`;
|
||||
const report = { generatedAt: new Date().toISOString(), platform: `${os.platform()} ${os.arch()}`, cpus: os.cpus().length, node: process.version,
|
||||
seedRows: count, seedMs, concurrency, allTimeSummary: distribution(reports), allTimeProjects: distribution(projectReports),
|
||||
hotCompanyWrites: { activePolicies: 0, ...distribution(writes), writesPerSecond: writes.length / (writeMs / 1000) }, companyLockAcquisition: distribution(waits),
|
||||
budgetedCompanyWrites: { activePolicies: 2, ...distribution(budgetedWrites), writesPerSecond: budgetedWrites.length / (budgetedMs / 1000) },
|
||||
admission: distribution(admissions), concurrentDashboard: distribution(dashboard), writesWhilePolling: distribution(writesWhilePolling),
|
||||
durableCheckpoint: distribution(checkpoints), healthWith100kHistoricalRuns: distribution(health),
|
||||
overviewWith102Policies: distribution(largeOverview),
|
||||
writesWith102Policies: { activePolicies: 102, relevantPolicies: 2, ...distribution(largePolicyWrites), writesPerSecond: largePolicyWrites.length / (largePolicyMs / 1000) },
|
||||
admissionWith102Policies: distribution(largePolicyAdmissions), spoolBacklogs,
|
||||
eventLoopDelay: { p95Ms: eventLoop.percentile(95) / 1e6, maxMs: eventLoop.max / 1e6 },
|
||||
recovery, projectBudgetQueryPlan: plan, integrityFindings: integrity.findings.length, totalMs: performance.now()-started };
|
||||
const output = path.resolve("coverage/accounting/scale.json"); await mkdir(path.dirname(output), { recursive: true }); await writeFile(output, JSON.stringify(report,null,2));
|
||||
process.stdout.write(JSON.stringify({ ...report, projectBudgetQueryPlan: "see coverage/accounting/scale.json" },null,2)+"\n");
|
||||
} finally {
|
||||
eventLoop.disable();
|
||||
try { await db.$client.end(); await database.cleanup(); }
|
||||
finally {
|
||||
if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome;
|
||||
await rm(benchmarkHome, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
import { costService, createCostEventInTransaction } from "../services/costs.js";
|
||||
import { withAccountingTransaction } from "../services/accounting-transaction.js";
|
||||
import { budgetService } from "../services/budgets.js";
|
||||
import { buildPaperclipRuntimeMcpServers } from "../services/heartbeat.js";
|
||||
import { resolveNativeRuntimeMcpSnapshot } from "../services/native-runtime/runtime-context.js";
|
||||
@@ -480,6 +482,69 @@ const actor = { actorType: "user" as const, actorId: "browser-reviewer" };
|
||||
f.access.refreshCatalog(f.connection.connectionId, actor),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
it("rolls back the Browser Use charge and Finance event if a spend projection fails", async () => {
|
||||
const f = await fixture();
|
||||
await costService(db).createEvent(f.company.id, { agentId: f.agent.id, provider: "fixture", model: "fixture", costCents: 100, occurredAt: new Date() });
|
||||
await f.service.execute(f.binding, f.grant, randomUUID(), "browser_start", { task: "Read example.com" });
|
||||
f.complete();
|
||||
await db.execute(sql`create function fail_browser_projection() returns trigger language plpgsql as $$ begin raise exception 'injected projection failure'; end $$`);
|
||||
await db.execute(sql.raw(`create trigger fail_browser_projection before update of spent_monthly_cents on companies for each row when (new.id = '${f.company.id}'::uuid) execute function fail_browser_projection()`));
|
||||
try {
|
||||
await f.tick();
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(1);
|
||||
expect(await db.select().from(financeEvents).where(eq(financeEvents.companyId, f.company.id))).toHaveLength(0);
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].spentMonthlyCents).toBe(100);
|
||||
expect((await db.select().from(browserUseRuns).where(eq(browserUseRuns.companyId, f.company.id)))[0].accountedCents).toBe(0);
|
||||
} finally {
|
||||
await db.execute(sql`drop trigger fail_browser_projection on companies`);
|
||||
await db.execute(sql`drop function fail_browser_projection()`);
|
||||
}
|
||||
await f.tick();
|
||||
await f.tick();
|
||||
expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("115.0000000");
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(2);
|
||||
expect(await db.select().from(financeEvents).where(eq(financeEvents.companyId, f.company.id))).toMatchObject([{ amountCents: 15 }]);
|
||||
expect((await db.select().from(companies).where(eq(companies.id, f.company.id)))[0].spentMonthlyCents).toBe(115);
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].spentMonthlyCents).toBe(115);
|
||||
});
|
||||
|
||||
it("settles Browser Use atomically behind another accounting writer without losing spend", async () => {
|
||||
const f = await fixture();
|
||||
const charge = (costCents: number) => ({ agentId: f.agent.id, provider: "fixture", model: "fixture", costCents, occurredAt: new Date() });
|
||||
await costService(db).createEvent(f.company.id, charge(100));
|
||||
await f.service.execute(f.binding, f.grant, randomUUID(), "browser_start", { task: "Read example.com" });
|
||||
f.complete();
|
||||
let release!: () => void;
|
||||
let locked!: () => void;
|
||||
const released = new Promise<void>(resolve => { release = resolve; });
|
||||
const ready = new Promise<void>(resolve => { locked = resolve; });
|
||||
const writer = withAccountingTransaction(db, f.company.id, async (tx, publications) => {
|
||||
await createCostEventInTransaction(tx, f.company.id, charge(5), publications);
|
||||
locked();
|
||||
await released;
|
||||
});
|
||||
await ready;
|
||||
const browser = f.tick();
|
||||
try {
|
||||
await vi.waitFor(async () => {
|
||||
const waiting = await db.execute(sql`select pid from pg_stat_activity where datname = current_database() and wait_event_type = 'Lock' and pid <> pg_backend_pid()`);
|
||||
expect(waiting.length).toBeGreaterThan(0);
|
||||
});
|
||||
// Nothing from Browser Use may become visible before its projections.
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(1);
|
||||
expect(await db.select().from(financeEvents).where(eq(financeEvents.companyId, f.company.id))).toHaveLength(0);
|
||||
} finally {
|
||||
release();
|
||||
await Promise.all([writer, browser]);
|
||||
}
|
||||
await f.tick(); // Replaying the provider's cumulative total is a no-op.
|
||||
expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("120.0000000");
|
||||
expect((await db.select().from(companies).where(eq(companies.id, f.company.id)))[0].spentMonthlyCents).toBe(120);
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].spentMonthlyCents).toBe(120);
|
||||
expect(await db.select().from(financeEvents).where(eq(financeEvents.companyId, f.company.id))).toMatchObject([{ amountCents: 15 }]);
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("observes delayed browser.ready, hides viewer credentials, accounts once and cleans up", async () => {
|
||||
const f = await fixture();
|
||||
const invocation = randomUUID();
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { AdapterUsageCheckpoint } from "@paperclipai/adapter-utils";
|
||||
import { priceCodexReceipt } from "../services/codex-pricing.js";
|
||||
|
||||
const receipt: AdapterUsageCheckpoint = {
|
||||
provider: "openai",
|
||||
biller: "openai",
|
||||
billingType: "metered_api",
|
||||
model: "gpt-6-astra",
|
||||
complete: true,
|
||||
usageBasis: "per_run",
|
||||
costUsd: null,
|
||||
usage: {
|
||||
inputTokens: 123536,
|
||||
cachedInputTokens: 1567209,
|
||||
outputTokens: 6625,
|
||||
},
|
||||
};
|
||||
describe("prospective Codex pricing", () => {
|
||||
it.each([
|
||||
["gpt-6-sol", "12.200000000"],
|
||||
["gpt-6.1-sol", "12.100000000"],
|
||||
["gpt-6-luna", "0.610000000"],
|
||||
["gpt-5.6-sol", "24.400000000"],
|
||||
])("uses the distinct published cache rate for %s", (model, expected) => {
|
||||
expect(
|
||||
priceCodexReceipt({
|
||||
...receipt,
|
||||
model,
|
||||
usage: {
|
||||
inputTokens: 1_000_000,
|
||||
cachedInputTokens: 1_000_000,
|
||||
outputTokens: 1_000_000,
|
||||
},
|
||||
}).costUsdExact,
|
||||
).toBe(expected);
|
||||
});
|
||||
it("prices the staging token shape without counting cached input twice", () => {
|
||||
const result = priceCodexReceipt(receipt);
|
||||
expect(result.costUsdExact).toBe("3.133819000");
|
||||
expect(result.costStatus).toBe("estimated");
|
||||
expect(result.pricingProvenance).toMatchObject({
|
||||
source: "rate_card",
|
||||
version: "openai-standard-2026-09-30",
|
||||
});
|
||||
expect(result.pricingProvenance?.evidence).toContain(
|
||||
"short per-request context assumed",
|
||||
);
|
||||
expect(receipt.costUsdExact).toBeUndefined();
|
||||
});
|
||||
it.each([
|
||||
{ complete: false },
|
||||
{ complete: false, usage: { inputTokens: 0, outputTokens: 0 } },
|
||||
{ costUsd: 0 },
|
||||
{ costUsd: 4.21 },
|
||||
{ costUsdExact: "0.004" },
|
||||
{ cacheAdjustedCostUsd: 0.01 },
|
||||
{ provider: "other" },
|
||||
{ biller: "proxy" },
|
||||
{ model: "gpt-6-astra-custom" },
|
||||
{ model: "unknown" },
|
||||
{ model: "constructor" },
|
||||
{ model: "__proto__" },
|
||||
{ usageBasis: undefined },
|
||||
{ usageBasis: null },
|
||||
{ billingType: "unknown" },
|
||||
{ billingType: "subscription_included" },
|
||||
{ usageBasis: "session_cumulative" },
|
||||
{ pricingContext: { serviceTier: "unsupported" } },
|
||||
{ usage: undefined },
|
||||
{ usage: { inputTokens: -1, outputTokens: 0 } },
|
||||
{ usage: { inputTokens: 1, outputTokens: 0, cacheWriteTokens: 2 } },
|
||||
])(
|
||||
"preserves reported costs and refuses ambiguous pricing: %j",
|
||||
(override) => {
|
||||
const input = { ...receipt, ...override } as AdapterUsageCheckpoint;
|
||||
expect(priceCodexReceipt(input)).toBe(input);
|
||||
},
|
||||
);
|
||||
it.each([
|
||||
["standard", "short", "61.000000000"],
|
||||
["fast", "short", "122.000000000"],
|
||||
["flex", "short", "30.500000000"],
|
||||
["standard", "long", "97.000000000"],
|
||||
["batch", "long", "48.500000000"],
|
||||
] as const)("handles %s/%s rates", (serviceTier, contextTier, expected) => {
|
||||
const result = priceCodexReceipt({
|
||||
...receipt,
|
||||
pricingContext: { serviceTier, contextTier },
|
||||
usage: {
|
||||
inputTokens: 1_000_000,
|
||||
cachedInputTokens: 1_000_000,
|
||||
outputTokens: 1_000_000,
|
||||
cacheWriteTokens: 0,
|
||||
},
|
||||
});
|
||||
expect(result.costUsdExact).toBe(expected);
|
||||
});
|
||||
it("charges cache writes once as a subset of input", () => {
|
||||
expect(
|
||||
priceCodexReceipt({
|
||||
...receipt,
|
||||
usage: {
|
||||
inputTokens: 1_000_000,
|
||||
cacheWriteTokens: 1_000_000,
|
||||
outputTokens: 0,
|
||||
},
|
||||
}).costUsdExact,
|
||||
).toBe("12.500000000");
|
||||
});
|
||||
it("retains nanodollar precision and does not reprice frozen receipts", () => {
|
||||
const priced = priceCodexReceipt({
|
||||
...receipt,
|
||||
model: "gpt-6-luna",
|
||||
usage: { inputTokens: 0, cachedInputTokens: 1, outputTokens: 0 },
|
||||
});
|
||||
expect(priced.costUsdExact).toBe("0.000000010");
|
||||
expect(priceCodexReceipt(priced)).toBe(priced);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,162 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import { agents, authUsers, companies, companyMemberships, costEvents, createDb, heartbeatRuns, issues, type Db } from "@paperclipai/db";
|
||||
import { costService } from "../services/costs.js";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
describe("cost attribution by user (PostgreSQL)", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>, db: Db;
|
||||
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("accounting-by-user-"); db = createDb(database.connectionString); }, 30_000);
|
||||
afterAll(async () => { await database?.cleanup(); });
|
||||
|
||||
async function fixture() {
|
||||
const [company] = await db.insert(companies).values({ name: "User costs", issuePrefix: `U${randomUUID().slice(0, 7)}` }).returning();
|
||||
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Shared worker", role: "engineer", adapterType: "process" }).returning();
|
||||
return { companyId: company.id, agentId: agent.id };
|
||||
}
|
||||
async function member(companyId: string, name: string, status = "active") {
|
||||
const userId = randomUUID();
|
||||
await db.insert(authUsers).values({ id: userId, name, email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() });
|
||||
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, status });
|
||||
return userId;
|
||||
}
|
||||
async function run(f: Awaited<ReturnType<typeof fixture>>, responsibleUserId: string | null) {
|
||||
return (await db.insert(heartbeatRuns).values({ ...f, responsibleUserId, status: "succeeded" }).returning())[0];
|
||||
}
|
||||
async function charge(f: Awaited<ReturnType<typeof fixture>>, overrides: Partial<typeof costEvents.$inferInsert> = {}) {
|
||||
await db.insert(costEvents).values({ ...f, provider: "openai", model: "fixture", costCents: 1,
|
||||
billingType: "metered_api", occurredAt: new Date(), ...overrides });
|
||||
}
|
||||
function trace() {
|
||||
const queries: string[] = [];
|
||||
const traced = drizzle(db.$client, { logger: { logQuery(query) { queries.push(query); } } }) as unknown as Db;
|
||||
return { service: costService(traced), ledgerReads: () => queries.filter(query => query.startsWith("select") && query.includes('from "cost_events"')) };
|
||||
}
|
||||
|
||||
it("reports spend for any number of users and only includes active humans with zero spend", async () => {
|
||||
const f = await fixture(), foreign = await fixture();
|
||||
const { service, ledgerReads } = trace();
|
||||
expect(await service.byUser(f.companyId)).toEqual({ activeUserCount: 0, rows: [] });
|
||||
await member(f.companyId, "Alice");
|
||||
await member(f.companyId, "Invited", "pending");
|
||||
await member(f.companyId, "Suspended", "suspended");
|
||||
await member(f.companyId, "Former", "archived");
|
||||
await member(foreign.companyId, "Foreign");
|
||||
await db.insert(companyMemberships).values({ companyId: f.companyId, principalType: "agent", principalId: f.agentId });
|
||||
await charge(f);
|
||||
expect(await service.byUser(f.companyId)).toMatchObject({ activeUserCount: 1, rows: [
|
||||
{ userId: null, costCents: 1 }, { userName: "Alice", costCents: 0 },
|
||||
] });
|
||||
expect(ledgerReads()).toHaveLength(2);
|
||||
await member(f.companyId, "Bob");
|
||||
expect(await service.byUser(f.companyId)).toMatchObject({ activeUserCount: 2, rows: [
|
||||
{ userId: null, costCents: 1 }, { userName: "Alice", costCents: 0 }, { userName: "Bob", costCents: 0 },
|
||||
] });
|
||||
expect(ledgerReads()).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("excludes the Board principal by ID and preserves its historical charges as unattributed", async () => {
|
||||
const f = await fixture();
|
||||
await db.insert(authUsers).values({ id: "local-board", name: "Board", email: "local@paperclip.local", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
|
||||
await db.insert(companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: "local-board" });
|
||||
const service = costService(db);
|
||||
expect(await service.byUser(f.companyId)).toEqual({ activeUserCount: 0, rows: [] });
|
||||
// A real person is never filtered by their display name.
|
||||
const human = await member(f.companyId, "Board");
|
||||
const humanRun = await run(f, human), boardRun = await run(f, "local-board");
|
||||
await charge(f, { heartbeatRunId: humanRun.id, costCents: 0.25 });
|
||||
await charge(f, { heartbeatRunId: boardRun.id, costCents: 1 });
|
||||
const report = await service.byUser(f.companyId);
|
||||
expect(report).toMatchObject({ activeUserCount: 1, rows: [
|
||||
{ userId: null, userName: null, costCentsExact: "1.0000000", runCount: 1 },
|
||||
{ userId: human, userName: "Board", costCentsExact: "0.2500000", runCount: 1 },
|
||||
] });
|
||||
expect(report.rows.some(row => row.userId === "local-board")).toBe(false);
|
||||
expect((await service.summary(f.companyId)).spendCentsExact).toBe("1.2500000");
|
||||
});
|
||||
|
||||
it("preserves exact spend, legacy cache semantics, zero-spend members and distinct runs", async () => {
|
||||
const f = await fixture();
|
||||
const alice = await member(f.companyId, "Alice"), bob = await member(f.companyId, "Bob");
|
||||
const former = await member(f.companyId, "Former", "archived");
|
||||
const a = await run(f, alice), b = await run(f, former);
|
||||
const [otherAgent] = await db.insert(agents).values({ companyId: f.companyId, name: "Second worker", role: "engineer", adapterType: "process" }).returning();
|
||||
const c = await run({ ...f, agentId: otherAgent.id }, alice);
|
||||
await charge(f, { heartbeatRunId: a.id, costCents: 0.1000001, inputTokens: 100, cachedInputTokens: 40, outputTokens: 5 });
|
||||
await charge(f, { heartbeatRunId: a.id, costCents: 0.2000002, receiptHash: "modern", costStatus: "estimated", inputTokens: 100, cachedInputTokens: 40, outputTokens: 10 });
|
||||
await charge(f, { heartbeatRunId: b.id, costCents: 0.4 });
|
||||
await charge(f, { heartbeatRunId: c.id, agentId: otherAgent.id, costCents: 0, costStatus: "unpriced", inputTokens: 9 });
|
||||
await charge(f, { heartbeatRunId: a.id, costCents: 0, costStatus: "unpriced", billingType: "subscription_included" });
|
||||
await charge(f, { costCents: 0.0000001 });
|
||||
const { service, ledgerReads } = trace();
|
||||
const report = await service.byUser(f.companyId, { allTime: true });
|
||||
expect(ledgerReads()).toHaveLength(1);
|
||||
expect(report.rows.find(row => row.userId === alice)).toMatchObject({
|
||||
costCentsExact: "0.3000003", eventCount: 4, estimatedEventCount: 1, unpricedEventCount: 1,
|
||||
inputTokens: 169, cachedInputTokens: 80, outputTokens: 15, runCount: 2,
|
||||
});
|
||||
expect(report.rows.find(row => row.userId === bob)).toMatchObject({ costCentsExact: "0.0000000", eventCount: 0, runCount: 0 });
|
||||
expect(report.rows.find(row => row.userId === former)).toMatchObject({ userName: "Former", costCentsExact: "0.4000000", runCount: 1 });
|
||||
expect(report.rows.find(row => row.userId === null)).toMatchObject({ costCentsExact: "0.0000001", runCount: 0 });
|
||||
const sum = report.rows.reduce((total, row) => total + BigInt(row.costCentsExact.replace(".", "")), 0n);
|
||||
expect(sum).toBe(7000004n);
|
||||
expect((await service.summary(f.companyId, { allTime: true })).spendCentsExact).toBe("0.7000004");
|
||||
});
|
||||
|
||||
it("retains charges and active memberships when user profiles are missing", async () => {
|
||||
const f = await fixture();
|
||||
const missing = [randomUUID(), randomUUID()].sort();
|
||||
const former = randomUUID();
|
||||
for (const userId of [...missing, former]) {
|
||||
await db.insert(companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: userId,
|
||||
status: userId === former ? "archived" : "active" });
|
||||
}
|
||||
const ownerRun = await run(f, former);
|
||||
await charge(f, { heartbeatRunId: ownerRun.id, costCents: 0.125 });
|
||||
const report = await costService(db).byUser(f.companyId);
|
||||
expect(report).toMatchObject({ activeUserCount: 2, rows: [
|
||||
{ userId: former, userName: "Former user", userImage: null, costCentsExact: "0.1250000", runCount: 1 },
|
||||
...missing.map(userId => ({ userId, userName: "Unknown user", userImage: null, costCentsExact: "0.0000000", runCount: 0 })),
|
||||
] });
|
||||
expect((await costService(db).summary(f.companyId)).spendCentsExact).toBe("0.1250000");
|
||||
});
|
||||
|
||||
it("keeps every company charge but never attributes through a foreign run, agent or user", async () => {
|
||||
const f = await fixture(), foreign = await fixture();
|
||||
const alice = await member(f.companyId, "Alice"); await member(f.companyId, "Bob");
|
||||
const outsider = await member(foreign.companyId, "Secret foreign name");
|
||||
const foreignRun = await run(foreign, alice);
|
||||
const wrongUser = await run(f, outsider);
|
||||
const unowned = await run(f, null);
|
||||
const [otherAgent] = await db.insert(agents).values({ companyId: f.companyId, name: "Other", role: "engineer", adapterType: "process" }).returning();
|
||||
const wrongAgent = await run({ ...f, agentId: otherAgent.id }, alice);
|
||||
for (const r of [foreignRun, wrongUser, unowned, wrongAgent]) await charge(f, { heartbeatRunId: r.id });
|
||||
await charge(foreign, { heartbeatRunId: foreignRun.id, costCents: 999 });
|
||||
const report = await costService(db).byUser(f.companyId, { allTime: true });
|
||||
expect(report.rows.find(row => row.userId === null)).toMatchObject({ costCentsExact: "4.0000000", eventCount: 4, runCount: 2 });
|
||||
expect(report.rows.find(row => row.userId === alice)?.costCents).toBe(0);
|
||||
expect(JSON.stringify(report)).not.toContain(outsider);
|
||||
expect(JSON.stringify(report)).not.toContain("Secret foreign name");
|
||||
});
|
||||
|
||||
it("filters receipt dates inclusively and never reassigns history when the issue owner changes", async () => {
|
||||
const f = await fixture();
|
||||
const alice = await member(f.companyId, "Alice"), bob = await member(f.companyId, "Bob");
|
||||
const [issue] = await db.insert(issues).values({ companyId: f.companyId, title: "Shared work", responsibleUserId: alice }).returning();
|
||||
const r = await run(f, alice);
|
||||
const from = new Date("2001-01-01T00:00:00Z"), to = new Date("2001-01-02T00:00:00Z");
|
||||
for (const occurredAt of [new Date(from.getTime() - 1), from, to, new Date(to.getTime() + 1)]) {
|
||||
await charge(f, { heartbeatRunId: r.id, issueId: issue.id, occurredAt });
|
||||
}
|
||||
await db.update(issues).set({ responsibleUserId: bob }).where(eq(issues.id, issue.id));
|
||||
const service = costService(db);
|
||||
expect((await service.byUser(f.companyId, { from, to })).rows.find(row => row.userId === alice)).toMatchObject({ costCents: 2, runCount: 1 });
|
||||
expect((await service.byUser(f.companyId, { allTime: true })).rows.find(row => row.userId === alice)?.costCents).toBe(4);
|
||||
expect((await service.byUser(f.companyId)).rows.every(row => row.costCents === 0)).toBe(true);
|
||||
await charge(f, { heartbeatRunId: r.id });
|
||||
expect((await service.byUser(f.companyId)).rows.find(row => row.userId === alice)?.costCents).toBe(1);
|
||||
await expect(service.byUser(randomUUID())).rejects.toThrow("Company not found");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createUsageCheckpointLog } from "@paperclipai/adapter-utils/usage-checkpoint";
|
||||
import { parseClaudeStreamJson } from "../../../packages/adapters/claude-local/src/server/parse.js";
|
||||
import { parseCodexJsonl } from "../../../packages/adapters/codex-local/src/server/parse.js";
|
||||
import { parseCursorJsonl } from "../../../packages/adapters/cursor-local/src/server/parse.js";
|
||||
import { parseGeminiJsonl } from "../../../packages/adapters/gemini-local/src/server/parse.js";
|
||||
import { parseOpenCodeJsonl } from "../../../packages/adapters/opencode-local/src/server/parse.js";
|
||||
import { parsePiJsonl } from "../../../packages/adapters/pi-local/src/server/parse.js";
|
||||
const secret = "TRANSCRIPT_CONTENT_MUST_NOT_ENTER_ACCOUNTING_BUFFER";
|
||||
const fixtures = [
|
||||
{ name: "claude", event: { type: "result", total_cost_usd: 0.0123, result: secret, usage: { input_tokens: 7, cache_read_input_tokens: 11, output_tokens: 3 } }, parse: parseClaudeStreamJson, input: 7, cached: 11, output: 3 },
|
||||
{ name: "codex", event: { type: "turn.completed", text: secret, usage: { input_tokens: 18, cached_input_tokens: 11, output_tokens: 3 } }, parse: parseCodexJsonl, input: 7, cached: 11, output: 3 },
|
||||
{ name: "cursor", event: { type: "result", result: secret, usage: { inputTokens: 7, cachedInputTokens: 11, outputTokens: 3 } }, parse: parseCursorJsonl, input: 7, cached: 11, output: 3 },
|
||||
{ name: "gemini", event: { type: "result", message: secret, stats: { input_tokens: 18, cached: 11, output_tokens: 1, total_tokens: 21 } }, parse: parseGeminiJsonl, input: 7, cached: 11, output: 3 },
|
||||
{ name: "opencode", event: { type: "step_finish", part: { text: secret, cost: 0.0123, tokens: { input: 5, cache: { read: 11, write: 2 }, output: 1, reasoning: 2 } } }, parse: parseOpenCodeJsonl, input: 7, cached: 11, output: 3 },
|
||||
{ name: "pi", event: { type: "turn_end", message: { role: "assistant", content: secret, usage: { input: 5, cacheWrite: 2, cacheRead: 11, output: 3, cost: { total: 0.0123 } } } }, parse: parsePiJsonl, input: 7, cached: 11, output: 3 },
|
||||
];
|
||||
describe("durable accounting stream projection", () => {
|
||||
it.each(fixtures)("captures split $name usage without retaining transcript content", async f => {
|
||||
const onUsage = vi.fn(async (..._args: any[]) => {}), onLog = vi.fn(async (..._args: any[]) => {});
|
||||
const parse = vi.fn((source: string) => {
|
||||
expect(source).not.toContain(secret);
|
||||
const result = f.parse(source); return { usage: result.usage ?? undefined, complete: true };
|
||||
});
|
||||
const log = createUsageCheckpointLog(onLog, onUsage, parse);
|
||||
const line = JSON.stringify(f.event) + "\n";
|
||||
for (let index = 0; index < line.length; index += 7) await log("stdout", line.slice(index,index+7));
|
||||
expect(onUsage).toHaveBeenCalledTimes(1);
|
||||
expect(onUsage.mock.calls[0][0]).toMatchObject({ complete: true, attemptId: expect.any(String), usage: { inputTokens: f.input, cachedInputTokens: f.cached, outputTokens: f.output } });
|
||||
expect(onLog.mock.calls.map(call => call[1]).join("")).toBe(line);
|
||||
});
|
||||
it("awaits durability before publication, suppresses identical snapshots and separates attempts", async () => {
|
||||
const order: string[] = [];
|
||||
const onUsage = vi.fn(async (..._args: any[]) => { order.push("durable"); });
|
||||
const parse = () => ({ usage: { inputTokens: 1, outputTokens: 2 }, complete: false });
|
||||
const log = createUsageCheckpointLog(async () => { order.push("published"); }, onUsage, parse);
|
||||
const message = '{"type":"result","usage":{"input_tokens":1}}\n';
|
||||
await log("stdout", message); await log("stdout", message);
|
||||
expect(order).toEqual(["durable", "published", "published"]);
|
||||
const other = createUsageCheckpointLog(async () => {}, onUsage, parse); await other("stdout", message);
|
||||
expect(onUsage.mock.calls[0][0].attemptId).not.toBe(onUsage.mock.calls[1][0].attemptId);
|
||||
});
|
||||
it("propagates receipt failures and leaves ordinary logs and stderr alone when disabled", async () => {
|
||||
const onLog = vi.fn(async (..._args: any[]) => {});
|
||||
const disabled = createUsageCheckpointLog(onLog, undefined, () => null);
|
||||
await disabled("stdout", "ordinary output"); await disabled.flush();
|
||||
expect(onLog).toHaveBeenCalledWith("stdout", "ordinary output");
|
||||
onLog.mockClear();
|
||||
const onUsage = vi.fn(async () => { throw new Error("spool full"); });
|
||||
const parse = vi.fn(() => ({ usage: { inputTokens: 1, outputTokens: 0 }, complete: false }));
|
||||
const log = createUsageCheckpointLog(onLog, onUsage, parse);
|
||||
await log("stderr", '{"usage":1}\n'); await log("stdout", 'invalid json\n{"text":"plain"}\n');
|
||||
expect(onUsage).not.toHaveBeenCalled(); onLog.mockClear();
|
||||
await log("stdout", '{"usage":{"input_tokens":1}}\n');
|
||||
expect(onLog).toHaveBeenCalled();
|
||||
await expect(log.flush()).rejects.toThrow("spool full");
|
||||
const oversized = createUsageCheckpointLog(onLog, onUsage, parse);
|
||||
await oversized("stdout", "x".repeat(8*1024*1024+1));
|
||||
await expect(oversized.flush()).rejects.toThrow("exceeds");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,764 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
afterAll,
|
||||
afterEach,
|
||||
beforeAll,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
vi,
|
||||
} from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import {
|
||||
agents,
|
||||
companies,
|
||||
companySecrets,
|
||||
costEvents,
|
||||
financeEvents,
|
||||
heartbeatRuns,
|
||||
providerBillingSnapshots,
|
||||
createDb,
|
||||
} from "@paperclipai/db";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { billingReconciliationService } from "../services/billing-reconciliation.js";
|
||||
import { financeService } from "../services/finance.js";
|
||||
import {
|
||||
applyProviderDailyCosts,
|
||||
importProviderDailyCosts,
|
||||
} from "../services/provider-billing-import.js";
|
||||
import * as secretsModule from "../services/secrets.js";
|
||||
import {
|
||||
createRunUsageRecorder,
|
||||
persistUsageReceipt,
|
||||
} from "../services/usage-receipts.js";
|
||||
import { accountRunCost } from "../services/run-cost-accounting.js";
|
||||
import { costService, getMonthlySpendTotal } from "../services/costs.js";
|
||||
import { budgetService } from "../services/budgets.js";
|
||||
import type { ImportProviderCosts } from "@paperclipai/shared";
|
||||
|
||||
describe("completed cost reporting paths (PostgreSQL)", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let spool: string;
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase(
|
||||
"accounting-completion-",
|
||||
);
|
||||
db = createDb(database.connectionString);
|
||||
spool = await mkdtemp(join(tmpdir(), "completion-spool-"));
|
||||
}, 90000);
|
||||
afterAll(async () => {
|
||||
await database?.cleanup();
|
||||
if (spool) await rm(spool, { recursive: true, force: true });
|
||||
});
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
const company = async () =>
|
||||
(
|
||||
await db
|
||||
.insert(companies)
|
||||
.values({
|
||||
name: "Completion",
|
||||
issuePrefix: `C${randomUUID().slice(0, 7)}`,
|
||||
})
|
||||
.returning()
|
||||
)[0];
|
||||
it("reads monthly spend from the ledger with company, agent and UTC month boundaries", async () => {
|
||||
const c = await company();
|
||||
const other = await company();
|
||||
const [first, second] = await db.insert(agents).values([
|
||||
{ companyId: c.id, name: "First", spentMonthlyCents: 9999 },
|
||||
{ companyId: c.id, name: "Second", spentMonthlyCents: 9999 },
|
||||
]).returning();
|
||||
const [outsider] = await db.insert(agents).values({ companyId: other.id, name: "Other" }).returning();
|
||||
const now = new Date();
|
||||
const start = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
|
||||
const end = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + 1, 1));
|
||||
const base = { companyId: c.id, agentId: first.id, provider: "test", model: "test", costCents: 1.125, occurredAt: start };
|
||||
await db.insert(costEvents).values([
|
||||
base,
|
||||
{ ...base, agentId: second.id, costCents: 2.25, occurredAt: new Date(end.getTime() - 1) },
|
||||
{ ...base, costCents: 500, occurredAt: new Date(start.getTime() - 1) },
|
||||
{ ...base, costCents: 600, occurredAt: end },
|
||||
{ ...base, companyId: other.id, agentId: outsider.id, costCents: 700 },
|
||||
]);
|
||||
expect(await getMonthlySpendTotal(db, { companyId: c.id })).toBe(3.375);
|
||||
expect(await getMonthlySpendTotal(db, { companyId: c.id, agentId: first.id })).toBe(1.125);
|
||||
expect(await getMonthlySpendTotal(db, { companyId: c.id, agentId: second.id })).toBe(2.25);
|
||||
expect(await getMonthlySpendTotal(db, { companyId: c.id, agentId: outsider.id })).toBe(0);
|
||||
expect(await getMonthlySpendTotal(db, { companyId: randomUUID() })).toBe(0);
|
||||
});
|
||||
|
||||
it("scopes agent and model estimate counts to the company and selected period", async () => {
|
||||
const c = await company();
|
||||
const other = await company();
|
||||
const [codie, bender] = await db.insert(agents).values([
|
||||
{ companyId: c.id, name: "Codie", adapterType: "paperclip_runner" },
|
||||
{ companyId: c.id, name: "Bender", adapterType: "claude_local" },
|
||||
]).returning();
|
||||
const [outsider] = await db.insert(agents).values({ companyId: other.id, name: "Other", adapterType: "process" }).returning();
|
||||
const base = { companyId: c.id, agentId: codie.id, provider: "openai", biller: "openai", billingType: "metered_api", model: "gpt-6-astra", costCents: 100, occurredAt: new Date("2026-09-10T12:00:00Z") };
|
||||
await db.insert(costEvents).values([
|
||||
{ ...base, costStatus: "estimated" },
|
||||
{ ...base, costStatus: "estimated" },
|
||||
{ ...base, model: "gpt-6-sol", costStatus: "reported" },
|
||||
{ ...base, costStatus: "reported", occurredAt: new Date("2026-08-31T23:59:59Z") },
|
||||
{ ...base, costStatus: "estimated", occurredAt: new Date("2026-10-01T00:00:00Z") },
|
||||
{ ...base, agentId: bender.id, costStatus: "reported" },
|
||||
{ ...base, companyId: other.id, agentId: outsider.id, costStatus: "estimated" },
|
||||
]);
|
||||
const range = { from: new Date("2026-09-01T00:00:00Z"), to: new Date("2026-09-30T23:59:59Z") };
|
||||
const service = costService(db);
|
||||
const totals = await service.byAgent(c.id, range);
|
||||
expect(totals).toHaveLength(2);
|
||||
expect(totals.find(row => row.agentId === codie.id)).toMatchObject({ costCents: 300, eventCount: 3, estimatedEventCount: 2, apiRunCount: 0 });
|
||||
expect(totals.find(row => row.agentId === bender.id)).toMatchObject({ eventCount: 1, estimatedEventCount: 0 });
|
||||
const models = (await service.byAgentModel(c.id, range)).filter(row => row.agentId === codie.id);
|
||||
expect(models).toHaveLength(2);
|
||||
expect(models.find(row => row.model === "gpt-6-astra")).toMatchObject({ eventCount: 2, estimatedEventCount: 2 });
|
||||
expect(models.find(row => row.model === "gpt-6-sol")).toMatchObject({ eventCount: 1, estimatedEventCount: 0 });
|
||||
});
|
||||
it("imports inference, fees and credits atomically into the timeline, without duplicating on retry", async () => {
|
||||
const c = await company();
|
||||
const service = billingReconciliationService(db);
|
||||
const invoice = {
|
||||
biller: "openai",
|
||||
externalId: "invoice-1",
|
||||
currency: "USD",
|
||||
lines: [
|
||||
{
|
||||
externalId: "usage",
|
||||
kind: "inference" as const,
|
||||
amountCents: "10.0000001",
|
||||
occurredAt: "2026-09-01T00:00:00Z",
|
||||
},
|
||||
{
|
||||
externalId: "subscription",
|
||||
kind: "fee" as const,
|
||||
amountCents: "2000",
|
||||
occurredAt: "2026-09-01T00:00:00Z",
|
||||
},
|
||||
{
|
||||
externalId: "refund",
|
||||
kind: "credit" as const,
|
||||
amountCents: "100",
|
||||
occurredAt: "2026-09-01T00:00:00Z",
|
||||
},
|
||||
],
|
||||
};
|
||||
const imports = await Promise.all([
|
||||
service.importInvoice(c.id, invoice, "board"),
|
||||
service.importInvoice(c.id, invoice, "board"),
|
||||
]);
|
||||
expect(imports[0].id).toBe(imports[1].id);
|
||||
expect(
|
||||
await financeService(db).list(c.id, {
|
||||
from: new Date("2026-09-01"),
|
||||
to: new Date("2026-09-30"),
|
||||
}),
|
||||
).toHaveLength(3);
|
||||
expect(
|
||||
(
|
||||
await financeService(db).summary(c.id, {
|
||||
from: new Date("2026-09-01"),
|
||||
to: new Date("2026-09-30"),
|
||||
})
|
||||
).netCentsExact,
|
||||
).toBe("1910.0000001");
|
||||
expect(
|
||||
await db.select().from(costEvents).where(eq(costEvents.companyId, c.id)),
|
||||
).toHaveLength(0);
|
||||
await expect(
|
||||
service.importInvoice(
|
||||
c.id,
|
||||
{ ...invoice, lines: [{ ...invoice.lines[0], amountCents: "99" }] },
|
||||
"board",
|
||||
),
|
||||
).rejects.toThrow("different contents");
|
||||
expect(
|
||||
await db
|
||||
.select()
|
||||
.from(financeEvents)
|
||||
.where(eq(financeEvents.companyId, c.id)),
|
||||
).toHaveLength(3);
|
||||
});
|
||||
it.each(["costEventId", "runId", "providerRequestId"] as const)("keeps overlapping invoice evidence out of Finance totals using %s", async (reference) => {
|
||||
const c = await company();
|
||||
const [agent] = await db.insert(agents).values({ companyId: c.id, name: "Browser worker" }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: c.id, agentId: agent.id, invocationSource: "on_demand" }).returning();
|
||||
const charge = await costService(db).createEvent(c.id, { agentId: agent.id, heartbeatRunId: run.id,
|
||||
provider: "browser-use-cloud", model: "test", providerRequestId: "request-1", costCents: "15.0000001", occurredAt: new Date() });
|
||||
const finance = financeService(db);
|
||||
await finance.createEvent(c.id, { costEventId: charge.id, heartbeatRunId: run.id,
|
||||
biller: "browser-use-cloud", eventKind: "inference_charge", amountCents: "15.0000001", occurredAt: new Date() });
|
||||
const service = billingReconciliationService(db);
|
||||
const linked = { [reference]: reference === "costEventId" ? charge.id : reference === "runId" ? run.id : "request-1" };
|
||||
const invoice = { biller: "browser-use-cloud", externalId: "invoice-1", currency: "USD", lines: [
|
||||
{ externalId: "usage", kind: "inference" as const, amountCents: "15.0000001", occurredAt: new Date().toISOString(), ...linked },
|
||||
] };
|
||||
const imported = await service.importInvoice(c.id, invoice, "board");
|
||||
expect((await service.reconcile(c.id, imported.id)).lines[0]).toMatchObject({ status: "matched", matchedEventId: charge.id });
|
||||
await service.importInvoice(c.id, invoice, "board");
|
||||
// A second export, even with a revised price, is evidence for review and
|
||||
// never a second debit for the same already-recorded purchase.
|
||||
const revised = await service.importInvoice(c.id, { ...invoice, externalId: "invoice-2", lines: [
|
||||
{ ...invoice.lines[0], amountCents: "16" },
|
||||
] }, "board");
|
||||
expect((await service.reconcile(c.id, revised.id)).lines[0].status).toBe("difference");
|
||||
expect(await finance.summary(c.id)).toMatchObject({ netCentsExact: "15.0000001", eventCount: 1 });
|
||||
expect(await finance.list(c.id)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("links a uniquely matched invoice debit and serializes overlapping exports", async () => {
|
||||
const c = await company();
|
||||
const [agent] = await db.insert(agents).values({ companyId: c.id, name: "Worker" }).returning();
|
||||
const charge = await costService(db).createEvent(c.id, { agentId: agent.id,
|
||||
provider: "test", model: "test", providerRequestId: "request", costCents: 15, occurredAt: new Date() });
|
||||
const service = billingReconciliationService(db);
|
||||
await Promise.all(["first-export", "second-export"].map(externalId => service.importInvoice(c.id, {
|
||||
biller: "test", externalId, currency: "USD", lines: [{ externalId: "usage", amountCents: "15",
|
||||
providerRequestId: "request", model: "test", occurredAt: new Date().toISOString() }],
|
||||
}, "board")));
|
||||
const rows = await financeService(db).list(c.id);
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].costEventId).toBe(charge.id);
|
||||
expect((await financeService(db).summary(c.id)).netCentsExact).toBe("15.0000000");
|
||||
});
|
||||
|
||||
it.each(["fee", "credit", "other currency", "other biller", "prior credit", "provider report"] as const)("preserves distinct invoice entries: %s", async (scenario) => {
|
||||
const c = await company();
|
||||
const [agent] = await db.insert(agents).values({ companyId: c.id, name: "Worker" }).returning();
|
||||
const charge = await costService(db).createEvent(c.id, { agentId: agent.id,
|
||||
provider: "test", model: "test", costCents: 15, occurredAt: new Date() });
|
||||
await financeService(db).createEvent(c.id, { costEventId: charge.id, biller: "test",
|
||||
eventKind: "inference_charge", direction: scenario === "prior credit" ? "credit" : "debit",
|
||||
amountCents: 15, occurredAt: new Date(),
|
||||
metadataJson: scenario === "provider report" ? { source: "provider_cost_report" } : undefined });
|
||||
const imported = await billingReconciliationService(db).importInvoice(c.id, {
|
||||
biller: scenario === "other biller" ? "other" : "test", externalId: "invoice",
|
||||
currency: scenario === "other currency" ? "EUR" : "USD",
|
||||
lines: [{ externalId: "line", costEventId: charge.id, amountCents: "2",
|
||||
kind: scenario === "fee" || scenario === "credit" ? scenario : "inference", occurredAt: new Date().toISOString() }],
|
||||
}, "board");
|
||||
const rows = await financeService(db).list(c.id);
|
||||
expect(rows).toHaveLength(2);
|
||||
expect(rows.find(row => row.metadataJson?.invoiceId === imported.id)).toMatchObject({
|
||||
amountCentsExact: "2.0000000", currency: scenario === "other currency" ? "EUR" : "USD",
|
||||
direction: scenario === "credit" ? "credit" : "debit",
|
||||
});
|
||||
});
|
||||
|
||||
it("retains ambiguous run evidence without guessing an additional debit", async () => {
|
||||
const c = await company();
|
||||
const [agent] = await db.insert(agents).values({ companyId: c.id, name: "Worker" }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: c.id, agentId: agent.id, invocationSource: "on_demand" }).returning();
|
||||
for (const costCents of [5, 10]) await costService(db).createEvent(c.id, {
|
||||
agentId: agent.id, heartbeatRunId: run.id, provider: "test", model: "test", costCents, occurredAt: new Date(),
|
||||
});
|
||||
const service = billingReconciliationService(db);
|
||||
const imported = await service.importInvoice(c.id, { biller: "test", externalId: "invoice", currency: "USD",
|
||||
lines: [{ externalId: "usage", runId: run.id, amountCents: "15", occurredAt: new Date().toISOString() }],
|
||||
}, "board");
|
||||
expect((await service.reconcile(c.id, imported.id)).lines[0].status).toBe("ambiguous");
|
||||
expect(await financeService(db).list(c.id)).toHaveLength(0);
|
||||
expect((await costService(db).summary(c.id)).spendCentsExact).toBe("15.0000000");
|
||||
});
|
||||
|
||||
it("does not select a price when multiple invoice lines claim the same charge", async () => {
|
||||
const c = await company();
|
||||
const [agent] = await db.insert(agents).values({ companyId: c.id, name: "Worker" }).returning();
|
||||
const charge = await costService(db).createEvent(c.id, { agentId: agent.id,
|
||||
provider: "test", model: "test", costCents: 15, occurredAt: new Date() });
|
||||
const service = billingReconciliationService(db);
|
||||
const imported = await service.importInvoice(c.id, { biller: "test", externalId: "invoice", currency: "USD",
|
||||
lines: [15, 16].map(amount => ({ externalId: `line-${amount}`, costEventId: charge.id,
|
||||
amountCents: String(amount), occurredAt: new Date().toISOString() })),
|
||||
}, "board");
|
||||
expect((await service.reconcile(c.id, imported.id)).lines.map(line => line.status)).toEqual(["ambiguous", "ambiguous"]);
|
||||
expect(await financeService(db).list(c.id)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("records provider report corrections as deltas, including decreases and returns to an earlier total", async () => {
|
||||
const c = await company();
|
||||
const input: ImportProviderCosts = {
|
||||
provider: "openai",
|
||||
accountId: "org_test",
|
||||
secretId: randomUUID(),
|
||||
scopeIds: ["project"],
|
||||
from: "2026-09-01",
|
||||
to: "2026-09-02",
|
||||
};
|
||||
const update = (amountCents: string) =>
|
||||
applyProviderDailyCosts(
|
||||
db,
|
||||
c.id,
|
||||
input,
|
||||
[{ day: "2026-09-01", scopeId: "project", amountCents }],
|
||||
"board",
|
||||
);
|
||||
await update("100");
|
||||
await update("100");
|
||||
await update("120");
|
||||
await update("90");
|
||||
await update("100");
|
||||
const rows = await financeService(db).list(c.id, {
|
||||
from: new Date("2026-09-01"),
|
||||
to: new Date("2026-09-30"),
|
||||
});
|
||||
expect(rows).toHaveLength(4);
|
||||
expect(rows.filter((row) => row.direction === "credit")).toHaveLength(1);
|
||||
const summary = await financeService(db).summary(c.id, { allTime: true });
|
||||
expect(summary).toMatchObject({
|
||||
netCentsExact: "0.0000000",
|
||||
providerReportedCentsExact: "100.0000000",
|
||||
});
|
||||
expect(await financeService(db).byBiller(c.id, { allTime: true })).toEqual(
|
||||
[],
|
||||
);
|
||||
expect(await financeService(db).byKind(c.id, { allTime: true })).toEqual(
|
||||
[],
|
||||
);
|
||||
const [snapshot] = await db
|
||||
.select()
|
||||
.from(providerBillingSnapshots)
|
||||
.where(eq(providerBillingSnapshots.companyId, c.id));
|
||||
expect(snapshot.amountCents).toBe("100.0000000");
|
||||
expect(snapshot.revision).toBe(4);
|
||||
await expect(
|
||||
applyProviderDailyCosts(
|
||||
db,
|
||||
c.id,
|
||||
input,
|
||||
[{ day: "2026-09-01", scopeId: "project", amountCents: "80" }],
|
||||
"board",
|
||||
new Map([["2026-09-01:project", 0]]),
|
||||
),
|
||||
).rejects.toThrow("newer provider report");
|
||||
const other = await company();
|
||||
await applyProviderDailyCosts(
|
||||
db,
|
||||
other.id,
|
||||
input,
|
||||
[{ day: "2026-09-01", scopeId: "project", amountCents: "1" }],
|
||||
"board",
|
||||
);
|
||||
expect(
|
||||
await db
|
||||
.select()
|
||||
.from(financeEvents)
|
||||
.where(eq(financeEvents.companyId, c.id)),
|
||||
).toHaveLength(4);
|
||||
});
|
||||
it.each([
|
||||
null,
|
||||
{},
|
||||
{ providerBilling: "openai" },
|
||||
{ providerBilling: { provider: "anthropic", accountId: "org_test" } },
|
||||
{ providerBilling: { provider: "openai", accountId: "org_other" } },
|
||||
{ providerBilling: { provider: "openai" } },
|
||||
])("never reads or transmits a secret without the matching billing designation: %j", async (providerMetadata) => {
|
||||
const c = await company();
|
||||
const [secret] = await db.insert(companySecrets).values({
|
||||
companyId: c.id, key: "unrelated", name: "Unrelated credential", providerMetadata,
|
||||
}).returning();
|
||||
const resolveSecretValue = vi.fn().mockResolvedValue("unrelated-private-key");
|
||||
vi.spyOn(secretsModule, "secretService").mockReturnValue({ resolveSecretValue } as unknown as ReturnType<typeof secretsModule.secretService>);
|
||||
const fetcher = vi.fn<typeof fetch>().mockResolvedValue(Response.json({ data: [], has_more: false }));
|
||||
await expect(importProviderDailyCosts(db, c.id, {
|
||||
provider: "openai", accountId: "org_test", secretId: secret.id,
|
||||
scopeIds: ["project"], from: "2026-09-01", to: "2026-09-02",
|
||||
}, "board", fetcher)).rejects.toThrow("designated for this billing provider and account");
|
||||
expect(resolveSecretValue).not.toHaveBeenCalled();
|
||||
expect(fetcher).not.toHaveBeenCalled();
|
||||
expect(await db.select().from(financeEvents).where(eq(financeEvents.companyId, c.id))).toEqual([]);
|
||||
});
|
||||
it("fetches with a company credential and keeps provider reports separate from invoice charges", async () => {
|
||||
const c = await company();
|
||||
const [secret] = await db
|
||||
.insert(companySecrets)
|
||||
.values({ companyId: c.id, key: "billing", name: "Billing admin", providerMetadata: { providerBilling: { provider: "openai", accountId: "org_test" } } })
|
||||
.returning();
|
||||
const resolveSecretValue = vi.fn().mockResolvedValue("fixture-admin-key");
|
||||
vi.spyOn(secretsModule, "secretService").mockReturnValue({
|
||||
resolveSecretValue,
|
||||
} as unknown as ReturnType<typeof secretsModule.secretService>);
|
||||
const fetcher = vi.fn<typeof fetch>().mockImplementation(async () =>
|
||||
Response.json({
|
||||
data: [
|
||||
{
|
||||
start_time: Date.parse("2026-09-01") / 1000,
|
||||
end_time: Date.parse("2026-09-02") / 1000,
|
||||
results: [
|
||||
{
|
||||
project_id: "project",
|
||||
amount: { value: 0.06, currency: "usd" },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
has_more: false,
|
||||
}),
|
||||
);
|
||||
const input: ImportProviderCosts = {
|
||||
provider: "openai",
|
||||
accountId: "org_test",
|
||||
secretId: secret.id,
|
||||
scopeIds: ["project"],
|
||||
from: "2026-09-01",
|
||||
to: "2026-09-02",
|
||||
};
|
||||
expect(
|
||||
await importProviderDailyCosts(db, c.id, input, "board", fetcher),
|
||||
).toEqual({ daysRead: 1, eventsCreated: 1 });
|
||||
expect(resolveSecretValue).toHaveBeenCalledWith(
|
||||
c.id,
|
||||
secret.id,
|
||||
"latest",
|
||||
expect.objectContaining({
|
||||
accessContext: expect.objectContaining({
|
||||
actorId: "board",
|
||||
consumerId: "provider-billing-import",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(fetcher.mock.calls[0][1]?.headers).toMatchObject({
|
||||
Authorization: "Bearer fixture-admin-key",
|
||||
"OpenAI-Organization": "org_test",
|
||||
});
|
||||
expect(
|
||||
await importProviderDailyCosts(db, c.id, input, "board", fetcher),
|
||||
).toEqual({ daysRead: 1, eventsCreated: 0 });
|
||||
await billingReconciliationService(db).importInvoice(
|
||||
c.id,
|
||||
{
|
||||
biller: "openai",
|
||||
externalId: "separate-invoice",
|
||||
currency: "USD",
|
||||
lines: [
|
||||
{
|
||||
externalId: "usage",
|
||||
amountCents: "6",
|
||||
kind: "inference",
|
||||
occurredAt: "2026-09-01T00:00:00Z",
|
||||
},
|
||||
],
|
||||
},
|
||||
"board",
|
||||
);
|
||||
expect(
|
||||
await financeService(db).summary(c.id, { allTime: true }),
|
||||
).toMatchObject({
|
||||
netCentsExact: "6.0000000",
|
||||
providerReportedCentsExact: "6.0000000",
|
||||
eventCount: 2,
|
||||
});
|
||||
expect(await financeService(db).byBiller(c.id, { allTime: true })).toEqual([
|
||||
expect.objectContaining({
|
||||
biller: "openai",
|
||||
netCentsExact: "6.0000000",
|
||||
eventCount: 1,
|
||||
}),
|
||||
]);
|
||||
const other = await company();
|
||||
await expect(
|
||||
importProviderDailyCosts(db, other.id, input, "board", fetcher),
|
||||
).rejects.toThrow("company billing credential");
|
||||
expect(resolveSecretValue).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
it("rejects a slow report when another importer changes the same provider snapshot", async () => {
|
||||
const c = await company();
|
||||
const [secret] = await db
|
||||
.insert(companySecrets)
|
||||
.values({ companyId: c.id, key: "billing-race", name: "Billing admin", providerMetadata: { providerBilling: { provider: "openai", accountId: "org_test" } } })
|
||||
.returning();
|
||||
vi.spyOn(secretsModule, "secretService").mockReturnValue({
|
||||
resolveSecretValue: vi.fn().mockResolvedValue("fixture-admin-key"),
|
||||
} as unknown as ReturnType<typeof secretsModule.secretService>);
|
||||
const input: ImportProviderCosts = {
|
||||
provider: "openai",
|
||||
accountId: "org_test",
|
||||
secretId: secret.id,
|
||||
scopeIds: ["project"],
|
||||
from: "2026-09-01",
|
||||
to: "2026-09-02",
|
||||
};
|
||||
const report = (value: number) =>
|
||||
Response.json({
|
||||
data: [
|
||||
{
|
||||
start_time: Date.parse(input.from) / 1000,
|
||||
end_time: Date.parse(input.to) / 1000,
|
||||
results: [
|
||||
{ project_id: "project", amount: { currency: "usd", value } },
|
||||
],
|
||||
},
|
||||
],
|
||||
has_more: false,
|
||||
});
|
||||
let release!: (response: Response) => void;
|
||||
let markStarted!: () => void;
|
||||
const slowResponse = new Promise<Response>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const started = new Promise<void>((resolve) => {
|
||||
markStarted = resolve;
|
||||
});
|
||||
const slowImport = importProviderDailyCosts(
|
||||
db,
|
||||
c.id,
|
||||
input,
|
||||
"board",
|
||||
async () => {
|
||||
markStarted();
|
||||
return slowResponse;
|
||||
},
|
||||
);
|
||||
await started;
|
||||
await importProviderDailyCosts(db, c.id, input, "board", async () =>
|
||||
report(2),
|
||||
);
|
||||
release(report(1));
|
||||
await expect(slowImport).rejects.toThrow("newer provider report");
|
||||
expect(
|
||||
await financeService(db).summary(c.id, { allTime: true }),
|
||||
).toMatchObject({
|
||||
providerReportedCentsExact: "200.0000000",
|
||||
eventCount: 1,
|
||||
});
|
||||
});
|
||||
it("rolls back an entire provider import when a later snapshot is invalid", async () => {
|
||||
const c = await company();
|
||||
const input: ImportProviderCosts = {
|
||||
provider: "openai",
|
||||
accountId: "org_test",
|
||||
secretId: randomUUID(),
|
||||
scopeIds: ["project"],
|
||||
from: "2026-09-01",
|
||||
to: "2026-09-03",
|
||||
};
|
||||
await expect(
|
||||
applyProviderDailyCosts(
|
||||
db,
|
||||
c.id,
|
||||
input,
|
||||
[
|
||||
{ day: "2026-09-01", scopeId: "project", amountCents: "5" },
|
||||
{ day: "2026-09-02", scopeId: "project", amountCents: "-1" },
|
||||
],
|
||||
"board",
|
||||
),
|
||||
).rejects.toThrow();
|
||||
expect(
|
||||
await db
|
||||
.select()
|
||||
.from(providerBillingSnapshots)
|
||||
.where(eq(providerBillingSnapshots.companyId, c.id)),
|
||||
).toEqual([]);
|
||||
expect(
|
||||
await db
|
||||
.select()
|
||||
.from(financeEvents)
|
||||
.where(eq(financeEvents.companyId, c.id)),
|
||||
).toEqual([]);
|
||||
});
|
||||
it("stores an explicit zero report and credits an amended report down to zero only once", async () => {
|
||||
const c = await company();
|
||||
const input: ImportProviderCosts = {
|
||||
provider: "anthropic",
|
||||
accountId: "org_test",
|
||||
secretId: randomUUID(),
|
||||
scopeIds: ["default"],
|
||||
from: "2026-09-01",
|
||||
to: "2026-09-02",
|
||||
};
|
||||
const update = (amountCents: string) =>
|
||||
applyProviderDailyCosts(
|
||||
db,
|
||||
c.id,
|
||||
input,
|
||||
[{ day: input.from, scopeId: "default", amountCents }],
|
||||
"board",
|
||||
);
|
||||
expect((await update("0")).eventsCreated).toBe(0);
|
||||
expect((await update("0")).eventsCreated).toBe(0);
|
||||
expect((await update("1.0000001")).eventsCreated).toBe(1);
|
||||
expect((await update("0")).eventsCreated).toBe(1);
|
||||
expect((await update("0")).eventsCreated).toBe(0);
|
||||
expect(
|
||||
await financeService(db).summary(c.id, { allTime: true }),
|
||||
).toMatchObject({
|
||||
providerReportedCentsExact: "0.0000000",
|
||||
netCentsExact: "0.0000000",
|
||||
eventCount: 2,
|
||||
});
|
||||
});
|
||||
it("freezes estimated dollars before finalization and persists them exactly once into costs and budgets", async () => {
|
||||
const c = await company();
|
||||
const budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(c.id, { scopeType: "company", scopeId: c.id, amount: 100 }, "board");
|
||||
const [agent] = await db
|
||||
.insert(agents)
|
||||
.values({
|
||||
companyId: c.id,
|
||||
name: "Codie",
|
||||
adapterType: "paperclip_runner",
|
||||
})
|
||||
.returning();
|
||||
const [run] = await db
|
||||
.insert(heartbeatRuns)
|
||||
.values({
|
||||
companyId: c.id,
|
||||
agentId: agent.id,
|
||||
invocationSource: "on_demand",
|
||||
status: "running",
|
||||
})
|
||||
.returning();
|
||||
const recorder = await createRunUsageRecorder(
|
||||
db,
|
||||
{ companyId: c.id, runId: run.id, adapterType: "paperclip_runner" },
|
||||
spool,
|
||||
);
|
||||
const captured = await recorder.capture({
|
||||
provider: "openai",
|
||||
biller: "openai",
|
||||
billingType: "metered_api",
|
||||
model: "gpt-6-astra",
|
||||
usageBasis: "per_run",
|
||||
complete: true,
|
||||
costUsd: null,
|
||||
usage: {
|
||||
inputTokens: 123536,
|
||||
cachedInputTokens: 1567209,
|
||||
outputTokens: 6625,
|
||||
},
|
||||
});
|
||||
expect(captured.costUsdExact).toBe("3.133819000");
|
||||
await db
|
||||
.update(heartbeatRuns)
|
||||
.set({ status: "failed", finishedAt: new Date() })
|
||||
.where(eq(heartbeatRuns.id, run.id));
|
||||
await accountRunCost(db, run.id);
|
||||
await accountRunCost(db, run.id);
|
||||
const [event] = await db
|
||||
.select()
|
||||
.from(costEvents)
|
||||
.where(eq(costEvents.heartbeatRunId, run.id));
|
||||
expect(event.costStatus).toBe("estimated");
|
||||
expect(event.pricingProvenance).toMatchObject({
|
||||
source: "rate_card",
|
||||
version: "openai-standard-2026-09-30",
|
||||
});
|
||||
const summary = await costService(db).summary(c.id);
|
||||
expect(summary.spendCentsExact).toBe("313.3819000");
|
||||
expect(summary.estimatedEventCount).toBe(1);
|
||||
expect(await budgets.getInvocationBlock(c.id, agent.id)).not.toBeNull();
|
||||
});
|
||||
it("preserves cache-write evidence and pricing across multiple provider attempts", async () => {
|
||||
const c = await company();
|
||||
const [agent] = await db
|
||||
.insert(agents)
|
||||
.values({
|
||||
companyId: c.id,
|
||||
name: "Retry",
|
||||
adapterType: "paperclip_runner",
|
||||
})
|
||||
.returning();
|
||||
const [run] = await db
|
||||
.insert(heartbeatRuns)
|
||||
.values({ companyId: c.id, agentId: agent.id, status: "running" })
|
||||
.returning();
|
||||
const recorder = await createRunUsageRecorder(
|
||||
db,
|
||||
{ companyId: c.id, runId: run.id, adapterType: "paperclip_runner" },
|
||||
spool,
|
||||
);
|
||||
await recorder.capture({
|
||||
attemptId: randomUUID(),
|
||||
complete: true,
|
||||
provider: "openai",
|
||||
biller: "openai",
|
||||
billingType: "api",
|
||||
model: "gpt-6-astra",
|
||||
usageBasis: "per_run",
|
||||
usage: { inputTokens: 10, cacheWriteTokens: 4, outputTokens: 2 },
|
||||
});
|
||||
const result = await recorder.capture({
|
||||
attemptId: randomUUID(),
|
||||
complete: true,
|
||||
provider: "openai",
|
||||
biller: "openai",
|
||||
billingType: "api",
|
||||
costUsd: 2,
|
||||
});
|
||||
expect(result).toMatchObject({
|
||||
complete: true,
|
||||
costUsdExact: "2.000210000",
|
||||
costStatus: "estimated",
|
||||
pricingProvenance: { source: "rate_card", version: "per-attempt/v1" },
|
||||
usage: {
|
||||
inputTokens: 10,
|
||||
cacheWriteTokens: 4,
|
||||
cachedInputTokens: 0,
|
||||
outputTokens: 2,
|
||||
},
|
||||
});
|
||||
await db
|
||||
.update(heartbeatRuns)
|
||||
.set({ status: "succeeded", finishedAt: new Date() })
|
||||
.where(eq(heartbeatRuns.id, run.id));
|
||||
await accountRunCost(db, run.id);
|
||||
expect(await costService(db).summary(c.id)).toMatchObject({
|
||||
spendCentsExact: "200.0210000",
|
||||
estimatedEventCount: 1,
|
||||
});
|
||||
});
|
||||
it("recovers older durable receipts without pricing metadata while preserving explicit zero", async () => {
|
||||
const c = await company();
|
||||
const sourceId = randomUUID();
|
||||
const [agent] = await db
|
||||
.insert(agents)
|
||||
.values({
|
||||
companyId: c.id,
|
||||
name: "Legacy receipt",
|
||||
adapterType: "process",
|
||||
})
|
||||
.returning();
|
||||
const [run] = await db
|
||||
.insert(heartbeatRuns)
|
||||
.values({
|
||||
companyId: c.id,
|
||||
agentId: agent.id,
|
||||
status: "running",
|
||||
usageJson: { accountingReceiptSourceId: sourceId },
|
||||
})
|
||||
.returning();
|
||||
const cases = [
|
||||
{ costUsd: null },
|
||||
{ costUsd: 0 },
|
||||
{ costUsdExact: "0.000000001" },
|
||||
];
|
||||
for (const [index, receipt] of cases.entries()) {
|
||||
await persistUsageReceipt(db, {
|
||||
schema: "paperclip/accounting-receipt/v1",
|
||||
id: randomUUID(),
|
||||
companyId: c.id,
|
||||
runId: run.id,
|
||||
adapterType: "process",
|
||||
sourceId,
|
||||
sequence: index + 1,
|
||||
receivedAt: new Date().toISOString(),
|
||||
receipt: { ...receipt, complete: true },
|
||||
});
|
||||
const [stored] = await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, run.id));
|
||||
expect(stored.usageJson?.pricingProvenance).toEqual({
|
||||
source: index === 0 ? "unknown" : "provider_reported",
|
||||
version: "accounting-receipt/v1",
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,113 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { agents, agentRuntimeState, approvals, budgetIncidents, budgetPolicies, companies, costEvents, createDb, heartbeatRuns, projects } from "@paperclipai/db";
|
||||
import { costService } from "../services/costs.js";
|
||||
import { budgetService, type BudgetServiceHooks } from "../services/budgets.js";
|
||||
import { accountRunCost, reconcileRunCosts } from "../services/run-cost-accounting.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
function random(seed: number) {
|
||||
return () => { seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0; return seed / 2 ** 32; };
|
||||
}
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("seeded concurrent accounting invariants", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("paperclip-accounting-interleavings-"); db = createDb(database.connectionString); }, 30_000);
|
||||
afterAll(async () => { await database?.cleanup(); });
|
||||
|
||||
it.each([1, 7, 42, 97, 31337, 65537, 104729, 0xdeadbeef])("conserves the ledger, totals, incidents and delivery state for seed %s", async (seed) => {
|
||||
const rng = random(seed);
|
||||
const [company] = await db.insert(companies).values({ name: `Interleaving ${seed}`, issuePrefix: `R${randomUUID().slice(0,7)}` }).returning();
|
||||
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", status: "idle", adapterType: "process" }).returning();
|
||||
const [project] = await db.insert(projects).values({ companyId: company.id, name: "Project" }).returning();
|
||||
const oracle = new Map<string, { ticks: bigint; input: number; cached: number; output: number; run: boolean }>();
|
||||
const operations: Array<() => Promise<unknown>> = [];
|
||||
let cancellationAttempts = 0;
|
||||
let successfulDeliveries = 0;
|
||||
const hooks: BudgetServiceHooks = { cancelWorkForScope: async (scope) => {
|
||||
expect(scope.companyId).toBe(company.id);
|
||||
expect(scope.createdBefore).toBeInstanceOf(Date);
|
||||
cancellationAttempts++;
|
||||
if (cancellationAttempts % 4 === 1) throw new Error(`Injected delivery interruption (seed ${seed})`);
|
||||
successfulDeliveries++;
|
||||
} };
|
||||
const budgets = budgetService(db, hooks);
|
||||
const costs = costService(db, hooks);
|
||||
const now = new Date();
|
||||
for (let index = 0; index < 12; index++) {
|
||||
const ticks = Math.floor(rng() * 9_000_000) + 1;
|
||||
const input = Math.floor(rng() * 300), cached = Math.floor(rng() * 500), output = Math.floor(rng() * 80);
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id,
|
||||
status: ["succeeded", "failed", "cancelled", "timed_out"][index % 4], finishedAt: now, costAccountingPending: true,
|
||||
usageJson: { accountingReceiptReady: true, provider: "fixture", model: `model-${index % 3}`, billingType: "metered_api", inputTokens: input, cachedInputTokens: cached, outputTokens: output, costUsd: ticks / 1e9, ledgerScope: { projectId: project.id } },
|
||||
}).returning();
|
||||
oracle.set(`heartbeat:${run.id}:final`, { ticks: BigInt(ticks), input, cached, output, run: true });
|
||||
operations.push(() => accountRunCost(db, run.id, hooks), () => accountRunCost(db, run.id, hooks));
|
||||
}
|
||||
for (let index = 0; index < 10; index++) {
|
||||
const ticks = Math.floor(rng() * 3_000_000) + 1;
|
||||
const key = `manual-${index}`;
|
||||
const receipt = { idempotencyKey: key, agentId: agent.id, projectId: project.id, provider: "fixture", model: "manual", billingType: "metered_api", costCents: ticks / 1e7, occurredAt: now };
|
||||
oracle.set(key, { ticks: BigInt(ticks), input: 0, cached: 0, output: 0, run: false });
|
||||
operations.push(() => costs.createEvent(company.id, receipt), () => costs.createEvent(company.id, receipt));
|
||||
}
|
||||
for (let index = 0; index < 12; index++) {
|
||||
const amount = rng() < 0.5 ? 1 : 200;
|
||||
const scopeType = index % 2 ? "agent" as const : "project" as const;
|
||||
operations.push(() => budgets.upsertPolicy(company.id, { scopeType, scopeId: scopeType === "agent" ? agent.id : project.id, amount, notifyEnabled: false }, "board"));
|
||||
operations.push(() => budgets.getInvocationBlock(company.id, agent.id, { projectId: project.id }));
|
||||
operations.push(() => reconcileRunCosts(db, hooks));
|
||||
}
|
||||
// Reproducible submission order and jitter; the database is free to choose
|
||||
// the actual interleaving. The oracle is independent of execution order.
|
||||
for (let i = operations.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rng() * (i + 1)); [operations[i], operations[j]] = [operations[j], operations[i]];
|
||||
}
|
||||
for (let offset = 0; offset < operations.length; offset += 8) {
|
||||
await Promise.all(operations.slice(offset, offset + 8).map(async operation => {
|
||||
await new Promise(resolve => setTimeout(resolve, Math.floor(rng() * 5)));
|
||||
return operation();
|
||||
}));
|
||||
const incidents = await db.select().from(budgetIncidents).where(and(eq(budgetIncidents.companyId, company.id), eq(budgetIncidents.status, "open")));
|
||||
expect(new Set(incidents.map(row => `${row.policyId}:${row.windowStart.toISOString()}:${row.thresholdType}`)).size).toBe(incidents.length);
|
||||
}
|
||||
await reconcileRunCosts(db, hooks);
|
||||
const rows = await db.select().from(costEvents).where(eq(costEvents.companyId, company.id));
|
||||
expect(rows).toHaveLength(oracle.size);
|
||||
for (const row of rows) {
|
||||
const expected = oracle.get(row.idempotencyKey!);
|
||||
expect(expected, `unexpected receipt ${row.idempotencyKey}; seed ${seed}`).toBeDefined();
|
||||
expect(BigInt(Math.round(row.costCents * 1e7))).toBe(expected!.ticks);
|
||||
expect([row.inputTokens, row.cachedInputTokens, row.outputTokens]).toEqual([expected!.input, expected!.cached, expected!.output]);
|
||||
}
|
||||
const total = [...oracle.values()].reduce((sum, row) => sum + row.ticks, 0n);
|
||||
expect(BigInt(Math.round((await costs.summary(company.id)).spendCents * 1e7))).toBe(total);
|
||||
expect(BigInt(Math.round((await costs.byProject(company.id))[0].costCents * 1e7))).toBe(total);
|
||||
const [companyProjection] = await db.select().from(companies).where(eq(companies.id, company.id));
|
||||
const [agentProjection] = await db.select().from(agents).where(eq(agents.id, agent.id));
|
||||
expect(BigInt(Math.round(companyProjection.spentMonthlyCents * 1e7))).toBe(total);
|
||||
expect(BigInt(Math.round(agentProjection.spentMonthlyCents * 1e7))).toBe(total);
|
||||
const [runtime] = await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, agent.id));
|
||||
const runs = [...oracle.values()].filter(row => row.run);
|
||||
expect(BigInt(Math.round(runtime.totalCostCents * 1e7))).toBe(runs.reduce((sum, row) => sum + row.ticks, 0n));
|
||||
expect([runtime.totalInputTokens,runtime.totalCachedInputTokens,runtime.totalOutputTokens])
|
||||
.toEqual(["input","cached","output"].map(key => runs.reduce((sum,row) => sum + Number(row[key as "input" | "cached" | "output"]),0)));
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, company.id))).every(run => !run.costAccountingPending && run.costAccountedAt !== null)).toBe(true);
|
||||
// Finish in a known hard-stop state, deliver failed effects, then grant more
|
||||
// budget. This checks the final observable state, not only receipt counts.
|
||||
await budgets.upsertPolicy(company.id, { scopeType: "agent", scopeId: agent.id, amount: 1, notifyEnabled: false }, "board");
|
||||
expect(await budgets.getInvocationBlock(company.id, agent.id)).toMatchObject({ scopeType: "agent" });
|
||||
for (let pass = 0; pass < 4; pass++) await budgets.deliverPendingEnforcement(company.id);
|
||||
expect(successfulDeliveries).toBeGreaterThan(0);
|
||||
const policies = await db.select().from(budgetPolicies).where(eq(budgetPolicies.companyId, company.id));
|
||||
expect(policies.every(row => row.enforcementVersion === row.enforcementDeliveredVersion)).toBe(true);
|
||||
const allIncidents = await db.select().from(budgetIncidents).where(eq(budgetIncidents.companyId, company.id));
|
||||
const approvalRows = await db.select().from(approvals).where(eq(approvals.companyId, company.id));
|
||||
expect(new Set(allIncidents.map(row => row.approvalId).filter(Boolean)).size).toBe(approvalRows.length);
|
||||
await budgets.upsertPolicy(company.id, { scopeType: "agent", scopeId: agent.id, amount: 200 }, "board");
|
||||
await budgets.upsertPolicy(company.id, { scopeType: "project", scopeId: project.id, amount: 200 }, "board");
|
||||
expect(await budgets.getInvocationBlock(company.id, agent.id, { projectId: project.id })).toBeNull();
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -0,0 +1,126 @@
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import { randomUUID, randomInt } from "node:crypto";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { createServer } from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { agents, companies, costEvents, heartbeatRuns, budgetPolicies, createDb } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
const root = fileURLToPath(new URL("../../..", import.meta.url));
|
||||
async function until<T>(read: () => Promise<T>, accept: (value: T) => boolean, diagnostic: () => string = () => "", timeout = 40_000) {
|
||||
const deadline = Date.now() + timeout;
|
||||
while (Date.now() < deadline) {
|
||||
const value = await read();
|
||||
if (accept(value)) return value;
|
||||
await new Promise(resolve => setTimeout(resolve, 30));
|
||||
}
|
||||
throw new Error(`Accounting process checkpoint timed out. ${diagnostic()}`);
|
||||
}
|
||||
|
||||
(support.supported && process.platform !== "win32" ? describe : describe.skip)("accounting survives real server SIGKILL", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("paperclip-accounting-crash-"); }, 30_000);
|
||||
afterAll(async () => { await database?.cleanup(); });
|
||||
|
||||
it.each(["before_receipt", "before_runtime_totals", "before_commit", "after_commit_before_delivery_ack"] as const)("recovers exactly once after %s", async (phase) => {
|
||||
const db = createDb(database.connectionString);
|
||||
const observer = createDb(database.connectionString).$client;
|
||||
const blocker = await createDb(database.connectionString).$client.reserve();
|
||||
const home = await mkdtemp(path.join(tmpdir(), "pc-accounting-kill-"));
|
||||
const key = randomInt(1, 2_000_000_000);
|
||||
const [company] = await db.insert(companies).values({ name: "Crash accounting", issuePrefix: `K${randomUUID().slice(0,7)}` }).returning();
|
||||
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", status: "idle", adapterType: "process", runtimeConfig: { heartbeat: { enabled: false } } }).returning();
|
||||
const [policy] = await db.insert(budgetPolicies).values({ companyId: company.id, scopeType: "agent", scopeId: agent.id, metric: "billed_cents", windowKind: "calendar_month_utc", amount: 1, notifyEnabled: false }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "failed", finishedAt: new Date(), costAccountingPending: true,
|
||||
usageJson: { accountingReceiptReady: true, provider: "fixture", model: "fixture", billingType: "metered_api", inputTokens: 7, cachedInputTokens: 11, outputTokens: 3, costUsd: 0.0125 },
|
||||
}).returning();
|
||||
const table = phase === "before_receipt" ? "cost_events" : phase === "before_runtime_totals" ? "agent_runtime_state"
|
||||
: phase === "before_commit" ? "heartbeat_runs" : "budget_policies";
|
||||
const event = phase === "after_commit_before_delivery_ack" || phase === "before_commit" ? "UPDATE" : "INSERT";
|
||||
const condition = phase === "after_commit_before_delivery_ack" ? "AND NEW.enforcement_delivered_version > OLD.enforcement_delivered_version"
|
||||
: phase === "before_commit" ? "AND OLD.cost_accounting_pending = false AND NEW.cost_accounted_at IS NOT NULL" : "";
|
||||
let child: ChildProcess | undefined;
|
||||
let logs = "";
|
||||
async function kill() {
|
||||
if (!child || child.exitCode !== null || child.signalCode !== null) return;
|
||||
const stopped = new Promise<void>((resolve) => child!.once("exit", () => resolve()));
|
||||
process.kill(-child.pid!, "SIGKILL");
|
||||
await stopped;
|
||||
}
|
||||
async function start() {
|
||||
const port = await new Promise<number>((resolve, reject) => {
|
||||
const listener = createServer(); listener.on("error", reject);
|
||||
listener.listen(0, "127.0.0.1", () => { const address = listener.address() as { port: number }; listener.close(() => resolve(address.port)); });
|
||||
});
|
||||
logs = "";
|
||||
child = spawn(process.execPath, ["cli/node_modules/tsx/dist/cli.mjs", "server/src/index.ts"], {
|
||||
cwd: root, detached: true, stdio: ["ignore", "pipe", "pipe"],
|
||||
env: { PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: process.env.TMPDIR,
|
||||
NODE_ENV: "test", DATABASE_URL: database.connectionString, PORT: String(port),
|
||||
PAPERCLIP_HOME: home, PAPERCLIP_CONFIG: path.join(home, "config.json"), PAPERCLIP_INSTANCE_ID: "accounting-crash",
|
||||
PAPERCLIP_DEPLOYMENT_MODE: "local_trusted", PAPERCLIP_BIND: "loopback", SERVE_UI: "false",
|
||||
PAPERCLIP_OPEN_ON_LISTEN: "false", PAPERCLIP_TELEMETRY_DISABLED: "1", PAPERCLIP_ANNOUNCEMENTS_ENABLED: "false",
|
||||
HEARTBEAT_SCHEDULER_ENABLED: "false", PAPERCLIP_DECISION_SIGNING_SECRET: "accounting-crash-test-decision-secret",
|
||||
PAPERCLIP_AGENT_JWT_SECRET: "accounting-crash-test-agent-secret",
|
||||
},
|
||||
});
|
||||
const capture = (data: Buffer) => { logs = (logs + data.toString()).slice(-16000); };
|
||||
child.stdout!.on("data", capture); child.stderr!.on("data", capture);
|
||||
return `http://127.0.0.1:${port}`;
|
||||
}
|
||||
try {
|
||||
await blocker`SELECT pg_advisory_lock(${key})`;
|
||||
await observer.unsafe(`CREATE FUNCTION accounting_crash_barrier() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN IF NEW.company_id = '${company.id}'::uuid ${condition} THEN PERFORM pg_advisory_lock(${key}); END IF; RETURN NEW; END $$;
|
||||
CREATE TRIGGER accounting_crash_barrier BEFORE ${event} ON ${table} FOR EACH ROW EXECUTE FUNCTION accounting_crash_barrier()`);
|
||||
await start();
|
||||
const waiting = await until(() => {
|
||||
if (child!.exitCode !== null || child!.signalCode !== null) throw new Error(`Accounting server exited before the barrier: ${logs}`);
|
||||
return observer`SELECT pid FROM pg_locks WHERE locktype='advisory' AND objid=${key} AND NOT granted`;
|
||||
}, rows => rows.length > 0, () => logs);
|
||||
const committed = phase === "after_commit_before_delivery_ack";
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, run.id))).toHaveLength(committed ? 1 : 0);
|
||||
await kill();
|
||||
expect(child?.signalCode).toBe("SIGKILL");
|
||||
// A backend blocked inside our artificial barrier cannot observe its
|
||||
// dead client until the query returns. Terminate that orphan connection
|
||||
// before releasing the barrier, forcing PostgreSQL's disconnect rollback.
|
||||
for (const row of waiting) await observer`SELECT pg_terminate_backend(${row.pid})`;
|
||||
await until(() => observer`SELECT pid FROM pg_locks WHERE locktype='advisory' AND objid=${key} AND NOT granted`, rows => rows.length === 0);
|
||||
await blocker`SELECT pg_advisory_unlock(${key})`;
|
||||
await observer.unsafe(`DROP TRIGGER accounting_crash_barrier ON ${table}; DROP FUNCTION accounting_crash_barrier()`);
|
||||
const baseUrl = await start();
|
||||
await until(() => observer`SELECT cost_accounting_pending,cost_accounted_at FROM heartbeat_runs WHERE id=${run.id}`,
|
||||
rows => rows[0].cost_accounting_pending === false && rows[0].cost_accounted_at !== null, () => logs);
|
||||
await until(() => observer`SELECT enforcement_version,enforcement_delivered_version FROM budget_policies WHERE id=${policy.id}`,
|
||||
rows => rows[0].enforcement_version > 0 && rows[0].enforcement_version === rows[0].enforcement_delivered_version, () => logs);
|
||||
const assertLedger = async () => {
|
||||
expect((await observer`SELECT count(*)::int AS count,sum(cost_cents)::text AS cents FROM cost_events WHERE heartbeat_run_id=${run.id}`)[0]).toEqual({ count: 1, cents: "1.2500000" });
|
||||
expect((await observer`SELECT total_input_tokens,total_cached_input_tokens,total_output_tokens,total_cost_cents::text FROM agent_runtime_state WHERE agent_id=${agent.id}`)[0])
|
||||
.toEqual({ total_input_tokens: "7", total_cached_input_tokens: "11", total_output_tokens: "3", total_cost_cents: "1.2500000" });
|
||||
expect((await observer`SELECT spent_monthly_cents::text,status FROM agents WHERE id=${agent.id}`)[0]).toEqual({ spent_monthly_cents: "1.2500000", status: "paused" });
|
||||
expect((await observer`SELECT count(*)::int AS count FROM budget_incidents WHERE policy_id=${policy.id}`)[0].count).toBe(1);
|
||||
expect((await observer`SELECT count(*)::int AS count FROM approvals WHERE company_id=${company.id}`)[0].count).toBe(1);
|
||||
};
|
||||
await assertLedger();
|
||||
await until(async () => fetch(`${baseUrl}/api/health`).then(r => r.ok).catch(() => false), Boolean, () => logs);
|
||||
await kill();
|
||||
const replayUrl = await start();
|
||||
await until(async () => fetch(`${replayUrl}/api/health`).then(r => r.ok).catch(() => false), Boolean, () => logs);
|
||||
await until(async () => logs.includes(`Server startup recovery complete on 127.0.0.1:${new URL(replayUrl).port}`), Boolean, () => logs);
|
||||
await assertLedger();
|
||||
} finally {
|
||||
await kill();
|
||||
await blocker`SELECT pg_advisory_unlock_all()`;
|
||||
blocker.release();
|
||||
await observer.unsafe(`DROP TRIGGER IF EXISTS accounting_crash_barrier ON ${table}; DROP FUNCTION IF EXISTS accounting_crash_barrier()`);
|
||||
await observer.end();
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
}, 150_000);
|
||||
});
|
||||
@@ -0,0 +1,241 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { activityLog, agents, companies, costEvents, createDb, goals, heartbeatRuns, issues, projects } from "@paperclipai/db";
|
||||
import { costService } from "../services/costs.js";
|
||||
import { financeService } from "../services/finance.js";
|
||||
import { budgetService } from "../services/budgets.js";
|
||||
import { accountRunCost, reconcileRunCosts } from "../services/run-cost-accounting.js";
|
||||
import { receiptFingerprint } from "../services/receipt-fingerprint.js";
|
||||
import { subscribeCompanyLiveEvents } from "../services/live-events.js";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("accounting validation and recovery edges (PostgreSQL)", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("paperclip-accounting-edges-"); db = createDb(database.connectionString); }, 30_000);
|
||||
afterAll(async () => { await database?.cleanup(); });
|
||||
async function fixture() {
|
||||
const [company] = await db.insert(companies).values({ name: "Accounting edges", issuePrefix: `E${randomUUID().slice(0, 7)}` }).returning();
|
||||
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process", status: "idle" }).returning();
|
||||
const [project] = await db.insert(projects).values({ companyId: company.id, name: "Project" }).returning();
|
||||
const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Work", projectId: project.id }).returning();
|
||||
const [goal] = await db.insert(goals).values({ companyId: company.id, title: "Goal" }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id }).returning();
|
||||
const receipt = { agentId: agent.id, provider: "fixture", model: "fixture", billingType: "metered_api", costCents: 0.1234567, occurredAt: new Date() };
|
||||
const cost = await costService(db).createEvent(company.id, receipt);
|
||||
const finance = { eventKind: "inference_charge", biller: "fixture", amountCents: 123, occurredAt: new Date() };
|
||||
return { company, agent, project, issue, goal, run, cost, receipt, finance,
|
||||
links: { agentId: agent.id, projectId: project.id, issueId: issue.id, goalId: goal.id, heartbeatRunId: run.id, costEventId: cost.id } };
|
||||
}
|
||||
|
||||
it.each(["agentId", "projectId", "issueId", "goalId", "heartbeatRunId", "costEventId"] as const)("rejects foreign and missing finance %s, accepts company-owned links", async field => {
|
||||
const a = await fixture(), b = await fixture();
|
||||
const service = financeService(db);
|
||||
await expect(service.createEvent(a.company.id, { ...a.finance, [field]: b.links[field] })).rejects.toMatchObject({ status: 422 });
|
||||
await expect(service.createEvent(a.company.id, { ...a.finance, [field]: randomUUID() })).rejects.toMatchObject({ status: 404 });
|
||||
expect(await service.list(a.company.id)).toHaveLength(0);
|
||||
const event = await service.createEvent(a.company.id, { ...a.finance, ...a.links }, { actorType: "user", actorId: "board", agentId: a.agent.id });
|
||||
expect(event[field]).toBe(a.links[field]);
|
||||
expect(await service.list(a.company.id)).toEqual([event]);
|
||||
});
|
||||
|
||||
it("normalizes finance retries and rolls back changed keys and invalid amounts/currencies", async () => {
|
||||
const f = await fixture(), service = financeService(db);
|
||||
const receipt = { ...f.finance, idempotencyKey: "invoice", currency: "usd", metadataJson: { entries: [{ z: 2, a: 1 }, null] } };
|
||||
const event = await service.createEvent(f.company.id, receipt);
|
||||
expect((await service.createEvent(f.company.id, { ...receipt, currency: "USD", metadataJson: { entries: [{ a: 1, z: 2 }, null] } })).id).toBe(event.id);
|
||||
await expect(service.createEvent(f.company.id, { ...receipt, amountCents: 124 })).rejects.toMatchObject({ status: 409 });
|
||||
for (const invalid of [{ amountCents: -1 }, { amountCents: "1.2.3" }, { amountCents: Number.POSITIVE_INFINITY }, { currency: "US" }]) {
|
||||
await expect(service.createEvent(f.company.id, { ...f.finance, ...invalid })).rejects.toMatchObject({ status: 422 });
|
||||
}
|
||||
expect(await service.list(f.company.id)).toHaveLength(1);
|
||||
expect((await db.select().from(activityLog).where(eq(activityLog.companyId, f.company.id))).filter(row => row.action === "finance_event.reported")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("keeps finance credits, estimates, currencies, dates, and companies separate in every report", async () => {
|
||||
const f = await fixture(), foreign = await fixture(), service = financeService(db);
|
||||
const date = new Date("2026-08-15T12:00:00Z");
|
||||
await service.createEvent(f.company.id, { ...f.finance, occurredAt: date, amountCents: 150, estimated: true });
|
||||
await service.createEvent(f.company.id, { ...f.finance, occurredAt: new Date(date.getTime() + 1), eventKind: "credit_refund", direction: "credit", amountCents: 40 });
|
||||
await service.createEvent(f.company.id, { ...f.finance, occurredAt: date, amountCents: 900, currency: "EUR" });
|
||||
await service.createEvent(f.company.id, { ...f.finance, occurredAt: new Date("2026-08-16T00:00:00Z"), amountCents: 999 });
|
||||
await service.createEvent(foreign.company.id, { ...foreign.finance, occurredAt: date, amountCents: 999 });
|
||||
const range = { from: date, to: new Date(date.getTime() + 1) };
|
||||
expect(await service.summary(f.company.id, range)).toMatchObject({ debitCents: 150, creditCents: 40, estimatedDebitCents: 150, netCents: 110, eventCount: 2,
|
||||
currencies: [expect.objectContaining({ currency: "EUR", netCents: 900, eventCount: 1 }), expect.objectContaining({ currency: "USD", netCents: 110, eventCount: 2 })] });
|
||||
expect(await service.byBiller(f.company.id, range)).toEqual([
|
||||
expect.objectContaining({ biller: "fixture", currency: "EUR", netCents: 900, eventCount: 1, kindCount: 1 }),
|
||||
expect.objectContaining({ biller: "fixture", currency: "USD", netCents: 110, eventCount: 2, kindCount: 2 }),
|
||||
]);
|
||||
expect(await service.byKind(f.company.id, range)).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ eventKind: "inference_charge", currency: "EUR", netCents: 900 }),
|
||||
expect.objectContaining({ eventKind: "inference_charge", currency: "USD", netCents: 150, estimatedDebitCents: 150 }),
|
||||
expect.objectContaining({ eventKind: "credit_refund", currency: "USD", netCents: -40, billerCount: 1 }),
|
||||
]));
|
||||
expect(await service.list(f.company.id, range, 1)).toEqual([expect.objectContaining({ direction: "credit", amountCents: 40 })]);
|
||||
expect(await service.summary(randomUUID())).toMatchObject({ currencies: [], debitCents: 0, creditCents: 0, netCents: 0, eventCount: 0 });
|
||||
});
|
||||
|
||||
it("rejects missing companies, missing links, run/agent mismatches, and invalid cost inputs without side effects", async () => {
|
||||
const f = await fixture(), costs = costService(db);
|
||||
await expect(costs.createEvent(randomUUID(), f.receipt)).rejects.toMatchObject({ status: 404 });
|
||||
await expect(costs.summary(randomUUID())).rejects.toMatchObject({ status: 404 });
|
||||
for (const field of ["agentId", "issueId", "projectId", "goalId", "heartbeatRunId"]) {
|
||||
await expect(costs.createEvent(f.company.id, { ...f.receipt, [field]: randomUUID() })).rejects.toMatchObject({ status: 404 });
|
||||
}
|
||||
const [otherAgent] = await db.insert(agents).values({ companyId: f.company.id, name: "Other", role: "engineer", adapterType: "process" }).returning();
|
||||
await expect(costs.createEvent(f.company.id, { ...f.receipt, agentId: otherAgent.id, heartbeatRunId: f.run.id })).rejects.toThrow("does not belong to agent");
|
||||
for (const invalid of [{ costCents: -1 }, { costCents: NaN }, { inputTokens: 0.5 }, { cachedInputTokens: -1 }, { outputTokens: 2 ** 31 }]) {
|
||||
await expect(costs.createEvent(f.company.id, { ...f.receipt, ...invalid })).rejects.toMatchObject({ status: 422 });
|
||||
}
|
||||
expect((await costs.summary(f.company.id)).spendCents).toBe(f.receipt.costCents);
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("aggregates billers with fractional cents, cache tokens, and subscription runs without cross-company leakage", async () => {
|
||||
const f = await fixture(), foreign = await fixture(), costs = costService(db);
|
||||
const date = new Date("2026-08-15T12:00:00Z");
|
||||
await costs.createEvent(f.company.id, { ...f.receipt, occurredAt: date, biller: "gateway", provider: "a", model: "a", heartbeatRunId: f.run.id, inputTokens: 2, cachedInputTokens: 3, outputTokens: 5 });
|
||||
await costs.createEvent(f.company.id, { ...f.receipt, occurredAt: date, biller: "gateway", provider: "b", model: "b", heartbeatRunId: f.run.id, billingType: "subscription_included", costCents: 0, inputTokens: 7, cachedInputTokens: 11, outputTokens: 13 });
|
||||
await costs.createEvent(foreign.company.id, { ...foreign.receipt, occurredAt: date, biller: "gateway", costCents: 999 });
|
||||
expect(await costs.byBiller(f.company.id, { from: date, to: date })).toEqual([{
|
||||
biller: "gateway", costCents: 0.1234567, costCentsExact: "0.1234567", inputTokens: 9, cachedInputTokens: 14, outputTokens: 18,
|
||||
apiRunCount: 1, subscriptionRunCount: 1, subscriptionInputTokens: 7, subscriptionCachedInputTokens: 11, subscriptionOutputTokens: 13, providerCount: 2, modelCount: 2,
|
||||
}]);
|
||||
});
|
||||
|
||||
it("commits a receipt and its audit record even when a live subscriber throws", async () => {
|
||||
const f = await fixture();
|
||||
const listener = vi.fn(() => { throw new Error("Injected disconnected live subscriber"); });
|
||||
const unsubscribe = subscribeCompanyLiveEvents(f.company.id, listener);
|
||||
const warning = vi.spyOn(logger, "warn").mockImplementation(() => undefined);
|
||||
try {
|
||||
const receipt = { ...f.receipt, idempotencyKey: "committed-before-publication" };
|
||||
const event = await costService(db).createEvent(f.company.id, receipt, { actorType: "user", actorId: "board" });
|
||||
expect((await costService(db).createEvent(f.company.id, receipt)).id).toBe(event.id);
|
||||
expect(listener).toHaveBeenCalledTimes(1);
|
||||
expect(warning).toHaveBeenCalledOnce();
|
||||
expect((await costService(db).summary(f.company.id)).spendCents).toBe(0.2469134);
|
||||
expect(await db.select().from(activityLog).where(eq(activityLog.entityId, event.id))).toHaveLength(1);
|
||||
} finally { unsubscribe(); warning.mockRestore(); }
|
||||
});
|
||||
|
||||
it("reports lifetime history consistently across every cost grouping and finance view", async () => {
|
||||
const f = await fixture(), costs = costService(db), finance = financeService(db);
|
||||
const occurredAt = new Date("2000-01-01T00:00:00Z");
|
||||
await costs.createEvent(f.company.id, { ...f.receipt, occurredAt, costCents: 5, projectId: f.project.id, inputTokens: 2 });
|
||||
await finance.createEvent(f.company.id, { ...f.finance, occurredAt });
|
||||
const lifetime = { allTime: true };
|
||||
for (const read of [costs.byAgent, costs.byAgentModel, costs.byProvider, costs.byBiller, costs.byProject]) {
|
||||
const rows = await read(f.company.id, lifetime);
|
||||
expect(rows.reduce((sum, row) => sum + row.costCents, 0)).toBe(5.1234567);
|
||||
expect(rows.reduce((sum, row) => sum + row.inputTokens, 0)).toBe(2);
|
||||
}
|
||||
expect((await costs.summary(f.company.id)).spendCents).toBe(0.1234567);
|
||||
expect((await costs.summary(f.company.id, lifetime)).spendCents).toBe(5.1234567);
|
||||
expect((await finance.summary(f.company.id)).eventCount).toBe(0);
|
||||
expect((await finance.summary(f.company.id, lifetime)).debitCents).toBe(123);
|
||||
expect(await finance.list(f.company.id, lifetime)).toHaveLength(1);
|
||||
expect((await finance.byBiller(f.company.id, lifetime))[0].netCents).toBe(123);
|
||||
expect((await finance.byKind(f.company.id, lifetime))[0].netCents).toBe(123);
|
||||
await db.update(companies).set({ budgetMonthlyCents: 1 }).where(eq(companies.id, f.company.id));
|
||||
expect((await costs.summary(f.company.id)).utilizationPercent).toBe(12.35);
|
||||
});
|
||||
|
||||
it.each(["company", "agent", "project"] as const)("rejects nonexistent and foreign %s budget scopes without creating policies", async scopeType => {
|
||||
const f = await fixture(), foreign = await fixture(), budgets = budgetService(db);
|
||||
await expect(budgets.upsertPolicy(f.company.id, { scopeType, scopeId: randomUUID(), amount: 10 }, "board")).rejects.toMatchObject({ status: 404 });
|
||||
await expect(budgets.upsertPolicy(f.company.id, { scopeType, scopeId: foreign[scopeType].id, amount: 10 }, "board")).rejects.toMatchObject({ status: 422 });
|
||||
expect(await budgets.listPolicies(f.company.id)).toHaveLength(0);
|
||||
if (scopeType !== "company") {
|
||||
await expect(budgets.getInvocationBlock(f.company.id, scopeType === "agent" ? foreign.agent.id : f.agent.id,
|
||||
scopeType === "project" ? { projectId: foreign.project.id } : undefined)).rejects.toMatchObject({ status: 404 });
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["unpriced", "pending"] as const)("refuses an incident budget raise while accounting is %s, without partial mutations", async kind => {
|
||||
const f = await fixture(), budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 10 }, "board");
|
||||
if (kind === "unpriced") {
|
||||
await costService(db).createEvent(f.company.id, { ...f.receipt, costStatus: "unpriced", costCents: 0 });
|
||||
} else {
|
||||
await db.update(heartbeatRuns).set({ status: "failed", costAccountingPending: true, finishedAt: new Date(), usageJson: { accountingReceiptReady: false } }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
await budgets.getInvocationBlock(f.company.id, f.agent.id);
|
||||
}
|
||||
const [incident] = (await budgets.overview(f.company.id)).activeIncidents;
|
||||
const resolution = { action: "raise_budget_and_resume" as const, amount: 100 };
|
||||
await expect(budgets.resolveIncident(f.company.id, randomUUID(), resolution, "board")).rejects.toMatchObject({ status: 404 });
|
||||
const foreign = await fixture();
|
||||
await expect(budgets.resolveIncident(foreign.company.id, incident.id, resolution, "board")).rejects.toMatchObject({ status: 404 });
|
||||
await expect(budgets.resolveIncident(f.company.id, incident.id, resolution, "board")).rejects.toThrow(kind === "unpriced" ? /unpriced usage/ : /finish accounting/);
|
||||
expect((await budgets.listPolicies(f.company.id))[0].amount).toBe(10);
|
||||
expect((await budgets.overview(f.company.id)).activeIncidents).toEqual([expect.objectContaining({ id: incident.id, status: "open" })]);
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("paused");
|
||||
});
|
||||
|
||||
it.each(["company", "project"] as const)("preserves an operator's %s pause even without a policy", async scopeType => {
|
||||
const f = await fixture();
|
||||
if (scopeType === "company") await db.update(companies).set({ status: "paused", pauseReason: "manual", pausedAt: new Date() }).where(eq(companies.id, f.company.id));
|
||||
else await db.update(projects).set({ pauseReason: "manual", pausedAt: new Date() }).where(eq(projects.id, f.project.id));
|
||||
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id, { projectId: f.project.id }))
|
||||
.toMatchObject({ scopeType, reason: expect.stringContaining("paused and cannot start") });
|
||||
});
|
||||
|
||||
it("resolves a company incident and updates the legacy monthly budget together", async () => {
|
||||
const f = await fixture(), budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 1 }, "board");
|
||||
await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 2 });
|
||||
const [incident] = (await budgets.overview(f.company.id)).activeIncidents;
|
||||
await budgets.resolveIncident(f.company.id, incident.id, { action: "raise_budget_and_resume", amount: 10 }, "board");
|
||||
expect((await db.select().from(companies).where(eq(companies.id, f.company.id)))[0]).toMatchObject({ status: "active", pauseReason: null, budgetMonthlyCents: 10 });
|
||||
expect((await budgets.listPolicies(f.company.id))[0].amount).toBe(10);
|
||||
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
|
||||
});
|
||||
|
||||
it.each([false, true])("records subscription tokens at zero incremental charge (split=%s)", async split => {
|
||||
const f = await fixture();
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", costAccountingPending: true,
|
||||
usageJson: { billingType: "subscription_included", costUsd: 1, inputTokens: 3, cachedInputTokens: 5, outputTokens: 7,
|
||||
...(split ? { usageByModel: [{ model: "a", costUsd: 1, usage: { inputTokens: 3, cachedInputTokens: 5, outputTokens: 7 } }] } : {}),
|
||||
},
|
||||
}).where(eq(heartbeatRuns.id, f.run.id));
|
||||
expect(await accountRunCost(db, f.run.id)).toBe(true);
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toEqual([
|
||||
expect.objectContaining({ costCents: 0, costStatus: "reported", inputTokens: 3, cachedInputTokens: 5, outputTokens: 7, billingType: "subscription_included" }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("rotates a full poisoned recovery batch so the next valid receipt cannot starve", async () => {
|
||||
const f = await fixture(), foreign = await fixture();
|
||||
const poison = await db.insert(heartbeatRuns).values(Array.from({ length: 100 }, () => ({
|
||||
companyId: f.company.id, agentId: f.agent.id, status: "failed", costAccountingPending: true, updatedAt: new Date(0),
|
||||
usageJson: { accountingReceiptReady: true, costUsd: 0.01, ledgerScope: { projectId: foreign.project.id } },
|
||||
}))).returning();
|
||||
const [valid] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "succeeded", costAccountingPending: true,
|
||||
updatedAt: new Date(1), usageJson: { accountingReceiptReady: true, costUsd: 0.02 },
|
||||
}).returning();
|
||||
const error = vi.spyOn(logger, "error").mockImplementation(() => undefined);
|
||||
try {
|
||||
expect(await reconcileRunCosts(db)).toEqual({ scanned: 100, accounted: 0 });
|
||||
expect(await reconcileRunCosts(db)).toEqual({ scanned: 100, accounted: 1 });
|
||||
expect(error).toHaveBeenCalledTimes(199);
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, valid.id))).toEqual([expect.objectContaining({ costCents: 2 })]);
|
||||
const rows = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.company.id));
|
||||
expect(rows.filter(row => row.costAccountingPending)).toHaveLength(100);
|
||||
expect(rows.filter(row => poison.some(bad => bad.id === row.id)).every(row => row.costAccountedAt === null)).toBe(true);
|
||||
expect((await costService(db).summary(f.company.id)).spendCents).toBe(2.1234567);
|
||||
expect(await accountRunCost(db, valid.id)).toBe(false);
|
||||
expect(await accountRunCost(db, randomUUID())).toBe(false);
|
||||
} finally { error.mockRestore(); }
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
it("fingerprints nested receipts independently of object order while preserving array order and values", () => {
|
||||
const a = { items: [{ z: 2, a: 1 }, null], at: new Date("2026-01-01T00:00:00Z") };
|
||||
expect(receiptFingerprint(a)).toBe(receiptFingerprint({ at: a.at, items: [{ a: 1, z: 2 }, null] }));
|
||||
expect(receiptFingerprint(a)).not.toBe(receiptFingerprint({ ...a, items: [null, { z: 2, a: 1 }] }));
|
||||
expect(receiptFingerprint(a)).not.toBe(receiptFingerprint({ ...a, items: [{ z: 3, a: 1 }, null] }));
|
||||
});
|
||||
@@ -0,0 +1,45 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { agents, companies, createDb } from "@paperclipai/db";
|
||||
import { costService } from "../services/costs.js";
|
||||
import { budgetService } from "../services/budgets.js";
|
||||
import { accountingIntegrityService } from "../services/accounting-integrity.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("accounting mutation sentinels", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("accounting-mutation-"); db = createDb(database.connectionString); }, 30_000);
|
||||
afterAll(async () => { await database?.cleanup(); });
|
||||
async function fixture() {
|
||||
const [company] = await db.insert(companies).values({ name: "Mutation", issuePrefix: `M${randomUUID().slice(0,7)}` }).returning();
|
||||
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process", status: "idle" }).returning();
|
||||
return { company, agent, receipt: { agentId: agent.id, provider: "fixture", model: "fixture", costCents: "1.0000001", idempotencyKey: randomUUID(), occurredAt: new Date() } };
|
||||
}
|
||||
it("deduplicates a retried receipt", async () => {
|
||||
const f = await fixture(), service = costService(db);
|
||||
const first = await service.createEvent(f.company.id, f.receipt);
|
||||
const second = await service.createEvent(f.company.id, f.receipt);
|
||||
expect(second.id, "ACCOUNTING_ASSERTION deduplication").toBe(first.id);
|
||||
expect((await service.summary(f.company.id)).eventCount).toBe(1);
|
||||
});
|
||||
it("isolates company reporting", async () => {
|
||||
const f = await fixture(), other = await fixture(), service = costService(db);
|
||||
await service.createEvent(f.company.id, f.receipt);
|
||||
await service.createEvent(other.company.id, { ...other.receipt, costCents: 1000 });
|
||||
expect((await service.summary(f.company.id)).spendCentsExact, "ACCOUNTING_ASSERTION company").toBe("1.0000001");
|
||||
});
|
||||
it("conserves agent projections", async () => {
|
||||
const f = await fixture();
|
||||
await costService(db).createEvent(f.company.id, f.receipt);
|
||||
await costService(db).createEvent(f.company.id, { ...f.receipt, idempotencyKey: "second" });
|
||||
expect((await accountingIntegrityService(db).inspect(f.company.id)).findings, "ACCOUNTING_ASSERTION projection").toEqual([]);
|
||||
});
|
||||
it("stops at the exact budget boundary", async () => {
|
||||
const f = await fixture();
|
||||
const budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 1 }, "board");
|
||||
await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 1 });
|
||||
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id), "ACCOUNTING_ASSERTION threshold").not.toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,551 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { promises as fs } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { accountingRuntimeBaselines, agents, agentRuntimeState, billingInvoiceLines, budgetPolicies, budgetReservations, companies, costEvents, createDb, heartbeatRuns, nativeRunFinalizations, issues, projects, runUsageReceipts } from "@paperclipai/db";
|
||||
import { accountingIntegrityService } from "../services/accounting-integrity.js";
|
||||
import { billingReconciliationService } from "../services/billing-reconciliation.js";
|
||||
import { budgetService, type BudgetEnforcementScope } from "../services/budgets.js";
|
||||
import { reserveRunBudget } from "../services/budget-reservations.js";
|
||||
import { costService } from "../services/costs.js";
|
||||
import { financeService } from "../services/finance.js";
|
||||
import { accountRunCost, reconcileRunCosts } from "../services/run-cost-accounting.js";
|
||||
import { createRunUsageRecorder, persistUsageReceipt, replayUsageReceipts, spoolUsageReceipt, usageReceiptSpoolPath, type UsageReceiptEnvelope } from "../services/usage-receipts.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("accounting operational edge cases", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>, db: ReturnType<typeof createDb>, directory: string;
|
||||
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("accounting-operational-edges-"); db = createDb(database.connectionString); directory = await fs.mkdtemp(path.join(os.tmpdir(), "accounting-edges-")); },30_000);
|
||||
afterAll(async () => { vi.restoreAllMocks(); await database?.cleanup(); await fs.rm(directory, { recursive: true, force: true }); });
|
||||
async function fixture() {
|
||||
const [company] = await db.insert(companies).values({ name: "Edges", issuePrefix: `E${randomUUID().slice(0,7)}` }).returning();
|
||||
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process", status: "idle" }).returning();
|
||||
const [project] = await db.insert(projects).values({ companyId: company.id, name: "Work" }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "running", usageJson: {} }).returning();
|
||||
return { company, agent, project, run };
|
||||
}
|
||||
const usage = { inputTokens: 7, cachedInputTokens: 11, outputTokens: 3 };
|
||||
async function event(f: Awaited<ReturnType<typeof fixture>>, extras: Record<string, unknown> = {}) {
|
||||
return costService(db).createEvent(f.company.id, { agentId: f.agent.id, provider: "fixture", model: "model", costCents: "0.1234567", occurredAt: new Date(), ...extras });
|
||||
}
|
||||
it("initializes an old month once, excludes backdated events, and repairs current-month drift", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(companies).set({ spendMonthUtc: "2000-01-01", spentMonthlyCents: 999 }).where(eq(companies.id,f.company.id));
|
||||
await event(f); await event(f, { occurredAt: new Date("2000-01-01") }); await event(f);
|
||||
const [row] = await db.select().from(companies).where(eq(companies.id,f.company.id));
|
||||
expect(row.spentMonthlyCents).toBe(0.2469134);
|
||||
expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]);
|
||||
await db.update(agents).set({ spentMonthlyCents: 4 }).where(eq(agents.id,f.agent.id));
|
||||
const review = await accountingIntegrityService(db).inspect(f.company.id);
|
||||
await accountingIntegrityService(db).repair(f.company.id,review.fingerprint,"Repair drift","board");
|
||||
await event(f);
|
||||
expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]);
|
||||
});
|
||||
it("keeps fractional finance credits and net totals exact beyond number precision", async () => {
|
||||
const f = await fixture(), finance = financeService(db);
|
||||
const base = { biller: "fixture", eventKind: "inference_charge", occurredAt: new Date() };
|
||||
await finance.createEvent(f.company.id,{ ...base, amountCents: "9007199254740992.0000001" });
|
||||
await finance.createEvent(f.company.id,{ ...base, direction: "credit", amountCents: "9007199254740992" });
|
||||
expect((await finance.summary(f.company.id)).netCentsExact).toBe("0.0000001");
|
||||
expect((await finance.byBiller(f.company.id))[0].netCentsExact).toBe("0.0000001");
|
||||
expect((await finance.byKind(f.company.id))[0].netCentsExact).toBe("0.0000001");
|
||||
});
|
||||
it("checks agent and project capacity, preserves reservations through policy edits, and releases proven bootstrap failures", async () => {
|
||||
const f = await fixture(), budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(f.company.id,{ scopeType:"agent",scopeId:f.agent.id,amount:20,reservationCents:"3" },"board");
|
||||
await budgets.upsertPolicy(f.company.id,{ scopeType:"project",scopeId:f.project.id,windowKind:"lifetime",amount:10,reservationCents:"6" },"board");
|
||||
await budgets.upsertPolicy(f.company.id,{ scopeType:"company",scopeId:f.company.id,amount:20,reservationCents:"1" },"board");
|
||||
const reserved = await reserveRunBudget(db,f.company.id,f.run.id,f.project.id);
|
||||
expect(reserved.amountCents).toBe("6.0000000");
|
||||
await budgets.upsertPolicy(f.company.id,{ scopeType:"project",scopeId:f.project.id,windowKind:"lifetime",amount:10,reservationCents:"0" },"board");
|
||||
await budgets.upsertPolicy(f.company.id,{ scopeType:"agent",scopeId:f.agent.id,amount:20,reservationCents:"0" },"board");
|
||||
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId,f.run.id)))[0].amountCents).toBe("6.0000000");
|
||||
await db.update(heartbeatRuns).set({ status:"failed",resultJson:{executionRecovery:{providerWorkStarted:false}} }).where(eq(heartbeatRuns.id,f.run.id));
|
||||
await accountRunCost(db,f.run.id);
|
||||
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId,f.run.id)))[0].state).toBe("released");
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.companyId,f.company.id))).toEqual([]);
|
||||
});
|
||||
it.each(["retryable_failure", "observed"])("keeps native %s receipts open through same-run recovery", async (phase) => {
|
||||
const f = await fixture();
|
||||
const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Recoverable native work" }).returning();
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
await reserveRunBudget(db, f.company.id, f.run.id, null);
|
||||
await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: f.run.id, issueId: issue.id, phase });
|
||||
const input = { companyId: f.company.id, runId: f.run.id, adapterType: "paperclip_runner" };
|
||||
const spool = path.join(directory, randomUUID());
|
||||
const recorder = await createRunUsageRecorder(db, input, spool);
|
||||
await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 100, outputTokens: 10 }, costUsd: 0.1 });
|
||||
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
expect(await accountRunCost(db, f.run.id)).toBe(false);
|
||||
await reconcileRunCosts(db);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountedAt).toBeNull();
|
||||
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("held");
|
||||
await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
const resumed = await createRunUsageRecorder(db, input, spool);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.accountingReceiptReady).toBe(false);
|
||||
const empty = await resumed.complete({ exitCode: 0, signal: null, timedOut: false });
|
||||
expect(empty).toMatchObject({ complete: false, usage: { inputTokens: 100, outputTokens: 10 } });
|
||||
// The provider's native delta is cumulative for this run across recovery.
|
||||
await resumed.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 150, outputTokens: 15 }, costUsd: 0.15 });
|
||||
await db.update(nativeRunFinalizations).set({ phase: "terminal_failure" }).where(eq(nativeRunFinalizations.runId, f.run.id));
|
||||
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
expect(await accountRunCost(db, f.run.id)).toBe(true);
|
||||
expect(await accountRunCost(db, f.run.id)).toBe(false);
|
||||
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id)))[0]).toMatchObject({ inputTokens: 150, outputTokens: 15, costCents: 15 });
|
||||
});
|
||||
|
||||
it.each([false, true])("recovers pending native receipts beyond the startup batch before replacement (save failure: %s)", async (failSave) => {
|
||||
const f = await fixture(), other = await fixture();
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "native" }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
await reserveRunBudget(db, f.company.id, f.run.id, null);
|
||||
const input = { companyId: f.company.id, runId: f.run.id, adapterType: "paperclip_runner" };
|
||||
const spool = path.join(directory, randomUUID());
|
||||
const recorder = await createRunUsageRecorder(db, input, spool);
|
||||
await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 100, outputTokens: 10 }, costUsd: 0.1 });
|
||||
const before = (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0];
|
||||
const unavailable = vi.spyOn(db, "transaction").mockRejectedValue(new Error("Database unavailable"));
|
||||
try {
|
||||
await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 150, outputTokens: 15 }, costUsd: 0.15 });
|
||||
await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 200, outputTokens: 20 }, costUsd: 0.2 });
|
||||
} finally { unavailable.mockRestore(); }
|
||||
const pending = (await fs.readdir(spool)).filter(name => name.endsWith(".json"));
|
||||
expect(pending).toHaveLength(2);
|
||||
for (const name of pending) await fs.rename(path.join(spool, name), path.join(spool, `zzz-${name}`));
|
||||
const unrelated: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", id: randomUUID(),
|
||||
companyId: other.company.id, runId: other.run.id, sourceId: randomUUID(), sequence: 1,
|
||||
receivedAt: new Date().toISOString(), adapterType: "process", receipt: { complete: false } };
|
||||
await Promise.all(Array.from({ length: 101 }, async (_, index) => {
|
||||
await fs.writeFile(path.join(spool, `000-${index}.json`), JSON.stringify({ ...unrelated, id: randomUUID(), sequence: index + 1 }));
|
||||
}));
|
||||
expect(await replayUsageReceipts(db, spool)).toEqual({ replayed: 100, failed: 0 });
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.inputTokens).toBe(100);
|
||||
if (failSave) {
|
||||
await db.execute(sql`create function reject_recovered_receipt() returns trigger language plpgsql as $$ begin raise exception 'receipt save failed'; end $$`);
|
||||
await db.execute(sql`create trigger reject_recovered_receipt before insert on run_usage_receipts for each row execute function reject_recovered_receipt()`);
|
||||
try {
|
||||
await expect(createRunUsageRecorder(db, input, spool)).rejects.toThrow();
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.accountingReceiptSourceId)
|
||||
.toBe(before.usageJson?.accountingReceiptSourceId);
|
||||
expect((await fs.readdir(spool)).filter(name => name.startsWith("zzz-"))).toHaveLength(2);
|
||||
} finally {
|
||||
await db.execute(sql`drop trigger reject_recovered_receipt on run_usage_receipts`);
|
||||
await db.execute(sql`drop function reject_recovered_receipt()`);
|
||||
}
|
||||
}
|
||||
const resumed = await createRunUsageRecorder(db, input, spool);
|
||||
expect((await fs.readdir(spool)).filter(name => name.startsWith("zzz-"))).toHaveLength(0);
|
||||
expect(await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, f.run.id))).toHaveLength(3);
|
||||
expect(await resumed.complete({ exitCode: 0, signal: null, timedOut: false }))
|
||||
.toMatchObject({ complete: false, usage: { inputTokens: 200, outputTokens: 20 } });
|
||||
await resumed.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 220, outputTokens: 22 }, costUsd: 0.22 });
|
||||
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
expect(await accountRunCost(db, f.run.id)).toBe(true);
|
||||
expect(await accountRunCost(db, f.run.id)).toBe(false);
|
||||
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id)))[0])
|
||||
.toMatchObject({ inputTokens: 220, outputTokens: 22, costCents: 22 });
|
||||
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("settled");
|
||||
expect(await replayUsageReceipts(db, spool)).toEqual({ replayed: 1, failed: 0 });
|
||||
expect((await costService(db).summary(f.company.id)).spendCents).toBe(22);
|
||||
});
|
||||
|
||||
it.each(["company", "agent", "project"] as const)("blocks fresh %s work without pausing native recovery", async (scopeType) => {
|
||||
const f = await fixture(), cancelWorkForScope = vi.fn(async (_scope: BudgetEnforcementScope) => {});
|
||||
const budgets = budgetService(db, { cancelWorkForScope });
|
||||
const scopeId = f[scopeType].id;
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: 100, reservationCents: "20" }, "board");
|
||||
const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Recover under budget" }).returning();
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
const original = await reserveRunBudget(db, f.company.id, f.run.id, f.project.id);
|
||||
await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: f.run.id, issueId: issue.id, phase: "retryable_failure" });
|
||||
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: f.run.id, adapterType: "paperclip_runner" }, path.join(directory, randomUUID()));
|
||||
await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 100, outputTokens: 10 }, costUsd: 0.1 });
|
||||
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
// Also recover a budget-owned pause left by the previous implementation.
|
||||
const table = { company: companies, agent: agents, project: projects }[scopeType];
|
||||
await db.update(table).set({ pauseReason: "budget", ...(scopeType === "project" ? {} : { status: "paused" }) }).where(eq(table.id, scopeId));
|
||||
await reconcileRunCosts(db, { cancelWorkForScope });
|
||||
expect((await db.select().from(table).where(eq(table.id, scopeId)))[0].pauseReason).toBeNull();
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("idle");
|
||||
expect(cancelWorkForScope.mock.calls.filter(([scope]) => scope.companyId === f.company.id)).toEqual([]);
|
||||
expect((await budgets.getInvocationBlock(f.company.id, f.agent.id, { projectId: f.project.id }))?.reason).toContain("native run recovers");
|
||||
const [fresh] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "running" }).returning();
|
||||
await expect(reserveRunBudget(db, f.company.id, fresh.id, f.project.id)).rejects.toThrow("native run recovers");
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountedAt).toBeNull();
|
||||
await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
await db.update(nativeRunFinalizations).set({ leaseOwner: "recovery", leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.run.id));
|
||||
expect(await reserveRunBudget(db, f.company.id, f.run.id, f.project.id, {}, "recovery"))
|
||||
.toMatchObject({ id: original.id, reused: true, amountCents: "20.0000000", state: "held" });
|
||||
expect(await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id))).toHaveLength(1);
|
||||
await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 150, outputTokens: 15 }, costUsd: 0.15 });
|
||||
await db.update(nativeRunFinalizations).set({ phase: "terminal_failure" }).where(eq(nativeRunFinalizations.runId, f.run.id));
|
||||
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
expect(await accountRunCost(db, f.run.id)).toBe(true);
|
||||
expect(await accountRunCost(db, f.run.id)).toBe(false);
|
||||
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id)))[0]).toMatchObject({ inputTokens: 150, costCents: 15 });
|
||||
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("settled");
|
||||
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id, { projectId: f.project.id })).toBeNull();
|
||||
});
|
||||
|
||||
it.each(["company", "agent", "project"] as const)("rechecks %s stops before native reservation reuse", async scopeType => {
|
||||
const f = await fixture(), budgets = budgetService(db), scopeId = f[scopeType].id;
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: 100, reservationCents: "20" }, "board");
|
||||
const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Retry after budget edit" }).returning();
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
const original = await reserveRunBudget(db, f.company.id, f.run.id, f.project.id);
|
||||
await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: f.run.id, issueId: issue.id, phase: "retryable_failure", leaseOwner: "retry", leaseExpiresAt: new Date(Date.now() + 60_000) });
|
||||
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
await event(f, { costCents: 50, projectId: f.project.id });
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: 40 }, "board");
|
||||
await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
await expect(reserveRunBudget(db, f.company.id, f.run.id, f.project.id, {}, "retry")).rejects.toThrow(/budget/);
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: 100 }, "board");
|
||||
expect(await reserveRunBudget(db, f.company.id, f.run.id, f.project.id, {}, "retry")).toMatchObject({ id: original.id, reused: true });
|
||||
const table = { company: companies, agent: agents, project: projects }[scopeType];
|
||||
await db.update(table).set({ pauseReason: "manual", ...(scopeType === "project" ? { pausedAt: new Date() } : { status: "paused" }) }).where(eq(table.id, scopeId));
|
||||
await expect(reserveRunBudget(db, f.company.id, f.run.id, f.project.id, {}, "retry")).rejects.toThrow(/paused/);
|
||||
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0]).toMatchObject({ id: original.id, state: "held", amountCents: "20.0000000" });
|
||||
});
|
||||
|
||||
it.each(["closed", "over_budget", "unpriced", "manual"])("preserves %s stops while native accounting is unfinished", async (stop) => {
|
||||
const f = await fixture(), budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100 }, "board");
|
||||
const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Pending native run" }).returning();
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
await reserveRunBudget(db, f.company.id, f.run.id, null);
|
||||
await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: f.run.id, issueId: issue.id, phase: stop === "closed" ? "terminal_failure" : "retryable_failure" });
|
||||
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
if (stop === "over_budget") await event(f, { costCents: 100 });
|
||||
if (stop === "unpriced") await event(f, { costCents: 0, costStatus: "unpriced" });
|
||||
if (stop === "manual") await db.update(agents).set({ status: "paused", pauseReason: "manual" }).where(eq(agents.id, f.agent.id));
|
||||
await budgets.reconcilePolicies();
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0])
|
||||
.toMatchObject({ status: "paused", pauseReason: stop === "manual" ? "manual" : "budget" });
|
||||
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("held");
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountedAt).toBeNull();
|
||||
});
|
||||
|
||||
it.each(["complete", "partial", "receipt_failure", "ledger_failure", "cleanup_failure", "moved", "unstable_spool"])("settles the newest run receipt beyond a full replay batch (%s)", async (scenario) => {
|
||||
const f = await fixture(), other = await fixture(), spool = usageReceiptSpoolPath();
|
||||
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: f.run.id, adapterType: "process" });
|
||||
await reserveRunBudget(db, f.company.id, f.run.id, null);
|
||||
await recorder.capture({ complete: true, usage: { inputTokens: 100, outputTokens: 10 }, costUsd: 0.1 });
|
||||
const unavailable = vi.spyOn(db, "transaction").mockRejectedValue(new Error("Database unavailable"));
|
||||
try { await recorder.capture({ complete: scenario !== "partial", usage: { inputTokens: 200, outputTokens: 20 }, costUsd: 0.2 }); }
|
||||
finally { unavailable.mockRestore(); }
|
||||
const [pending] = (await fs.readdir(spool)).filter(name => name.endsWith(".json"));
|
||||
const file = path.join(spool, `zzz-${pending}`);
|
||||
await fs.rename(path.join(spool, pending), file);
|
||||
const unrelated: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", id: randomUUID(), companyId: other.company.id,
|
||||
runId: other.run.id, sourceId: randomUUID(), sequence: 1, receivedAt: new Date().toISOString(), adapterType: "process", receipt: { complete: false } };
|
||||
for (let index = 0; index < 101; index++) await fs.writeFile(path.join(spool, `000-${index}.json`), JSON.stringify({ ...unrelated, id: randomUUID(), sequence: index + 1 }));
|
||||
expect(await replayUsageReceipts(db)).toEqual({ replayed: 100, failed: 0 });
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.inputTokens).toBe(100);
|
||||
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
if (scenario === "unstable_spool") {
|
||||
const stat = fs.lstat.bind(fs);
|
||||
const unstable = vi.spyOn(fs, "lstat").mockImplementation(async (target, ...args) => {
|
||||
if (target === file) throw Object.assign(new Error("Concurrent rename"), { code: "ENOENT" });
|
||||
return stat(target, ...args);
|
||||
});
|
||||
try { await expect(accountRunCost(db, f.run.id)).rejects.toThrow("retry required"); }
|
||||
finally { unstable.mockRestore(); }
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountedAt).toBeNull();
|
||||
expect(JSON.parse(await fs.readFile(file, "utf8")).receipt.usage.inputTokens).toBe(200);
|
||||
}
|
||||
if (scenario.endsWith("_failure") && scenario !== "cleanup_failure") {
|
||||
const table = scenario === "receipt_failure" ? "run_usage_receipts" : "cost_events";
|
||||
await db.execute(sql`create function reject_settlement() returns trigger language plpgsql as $$ begin raise exception 'settlement save failed'; end $$`);
|
||||
await db.execute(sql.raw(`create trigger reject_settlement before insert on ${table} for each row execute function reject_settlement()`));
|
||||
try {
|
||||
await expect(accountRunCost(db, f.run.id)).rejects.toThrow();
|
||||
expect(JSON.parse(await fs.readFile(file, "utf8")).receipt.usage.inputTokens).toBe(200);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0])
|
||||
.toMatchObject({ costAccountedAt: null, usageJson: { inputTokens: 100 } });
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toEqual([]);
|
||||
expect(await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, f.run.id))).toHaveLength(1);
|
||||
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("held");
|
||||
} finally {
|
||||
await db.execute(sql.raw(`drop trigger reject_settlement on ${table}`));
|
||||
await db.execute(sql`drop function reject_settlement()`);
|
||||
}
|
||||
}
|
||||
const remove = fs.rm.bind(fs), stat = fs.lstat.bind(fs);
|
||||
let moved = false;
|
||||
const rm = scenario === "cleanup_failure" ? vi.spyOn(fs, "rm").mockImplementation(async (target, options) => {
|
||||
if (target === file) throw new Error("Temporary cleanup failure");
|
||||
return remove(target, options);
|
||||
}) : null;
|
||||
const lstat = scenario === "moved" ? vi.spyOn(fs, "lstat").mockImplementation(async (target, ...args) => {
|
||||
if (target === file && !moved) { moved = true; await fs.rename(file, `${file.slice(0, -5)}-moved.json`); }
|
||||
return stat(target, ...args);
|
||||
}) : null;
|
||||
try { await reconcileRunCosts(db); } finally { rm?.mockRestore(); lstat?.mockRestore(); }
|
||||
if (scenario === "partial") {
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0])
|
||||
.toMatchObject({ costAccountedAt: null, usageJson: { inputTokens: 200, accountingReceiptReady: false } });
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toEqual([]);
|
||||
await recorder.capture({ complete: true, usage: { inputTokens: 300, outputTokens: 30 }, costUsd: 0.3 });
|
||||
expect(await accountRunCost(db, f.run.id)).toBe(true);
|
||||
}
|
||||
expect(await accountRunCost(db, f.run.id)).toBe(false);
|
||||
const expected = scenario === "partial" ? 300 : 200;
|
||||
const charges = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id));
|
||||
expect(charges).toHaveLength(1);
|
||||
expect(charges[0]).toMatchObject({ inputTokens: expected, outputTokens: expected / 10, costCents: expected / 10 });
|
||||
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("settled");
|
||||
expect(await replayUsageReceipts(db)).toEqual({ replayed: scenario === "cleanup_failure" ? 2 : 1, failed: 0 });
|
||||
expect((await costService(db).summary(f.company.id)).spendCents).toBe(expected / 10);
|
||||
expect(await fs.readdir(spool)).toEqual([]);
|
||||
});
|
||||
|
||||
it("leaves unrelated corrupt spool evidence alone and aborts handoff on unreadable or invalid matching receipts", async () => {
|
||||
const f = await fixture(), spool = path.join(directory, randomUUID());
|
||||
const input = { companyId: f.company.id, runId: f.run.id, adapterType: "paperclip_runner" };
|
||||
await createRunUsageRecorder(db, input, spool);
|
||||
const source = (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.accountingReceiptSourceId;
|
||||
const matching = path.join(spool, "matching.json");
|
||||
await fs.writeFile(matching, JSON.stringify({ ...input, receipt: "invalid" }));
|
||||
await expect(createRunUsageRecorder(db, input, spool)).rejects.toThrow();
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.accountingReceiptSourceId).toBe(source);
|
||||
const read = vi.spyOn(fs, "readFile").mockRejectedValueOnce(Object.assign(new Error("Unreadable receipt"), { code: "EACCES" }));
|
||||
try { await expect(createRunUsageRecorder(db, input, spool)).rejects.toThrow("Unreadable receipt"); } finally { read.mockRestore(); }
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.accountingReceiptSourceId).toBe(source);
|
||||
await fs.rm(matching);
|
||||
const unrelated = ["invalid", "null", "{}", JSON.stringify({ companyId: f.company.id }),
|
||||
JSON.stringify({ companyId: f.company.id, runId: randomUUID() }), "x".repeat(1024 * 1024 + 1)];
|
||||
for (const [index, value] of unrelated.entries()) await fs.writeFile(path.join(spool, `${index}.json`), value);
|
||||
await fs.mkdir(path.join(spool, "directory.json"));
|
||||
const stat = vi.spyOn(fs, "lstat").mockRejectedValueOnce(Object.assign(new Error("Concurrent replay removed file"), { code: "ENOENT" }));
|
||||
try { await createRunUsageRecorder(db, input, spool); } finally { stat.mockRestore(); }
|
||||
await createRunUsageRecorder(db, input, spool);
|
||||
expect(await fs.readdir(spool)).toHaveLength(unrelated.length + 1);
|
||||
});
|
||||
|
||||
it("indexes a recovery backlog once outside company locks and still sees newly published receipts", async () => {
|
||||
const f = await fixture(), other = await fixture(), spool = usageReceiptSpoolPath();
|
||||
const runs = [f.run];
|
||||
for (let i = 0; i < 7; i++) {
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "running" }).returning();
|
||||
runs.push(run);
|
||||
}
|
||||
for (const run of runs) {
|
||||
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "process" });
|
||||
await recorder.capture({ complete: true, usage: { inputTokens: 50, outputTokens: 5 }, costUsd: 0.05 });
|
||||
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
|
||||
}
|
||||
const [first] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id));
|
||||
const envelope: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", id: randomUUID(), companyId: other.company.id,
|
||||
runId: other.run.id, sourceId: randomUUID(), sequence: 1, receivedAt: new Date().toISOString(), adapterType: "process", receipt: { complete: false } };
|
||||
const foreignFiles = new Set<string>();
|
||||
for (let i = 0; i < 256; i++) foreignFiles.add(await spoolUsageReceipt({ ...envelope, id: randomUUID(), sequence: i + 1 }));
|
||||
const reads = new Map<string, number>(), read = fs.readFile.bind(fs);
|
||||
let checkedLock = false, lockWasAvailable = false;
|
||||
const spy = vi.spyOn(fs, "readFile").mockImplementation(async (file, options) => {
|
||||
if (typeof file === "string" && foreignFiles.has(file)) {
|
||||
reads.set(file, (reads.get(file) ?? 0) + 1);
|
||||
if (!checkedLock) {
|
||||
checkedLock = true;
|
||||
await db.transaction(async tx => {
|
||||
await tx.execute(sql`set local lock_timeout = '100ms'`);
|
||||
await tx.select().from(companies).where(eq(companies.id, f.company.id)).for("no key update");
|
||||
});
|
||||
lockWasAvailable = true;
|
||||
// Published after the index's directory snapshot: the locked drain
|
||||
// must discover it instead of trusting a stale batch inventory.
|
||||
await spoolUsageReceipt({ ...envelope, id: randomUUID(), companyId: f.company.id, runId: f.run.id,
|
||||
sourceId: String(first.usageJson?.accountingReceiptSourceId), sequence: 2,
|
||||
receipt: { complete: true, usage: { inputTokens: 200, outputTokens: 20 }, costUsd: 0.2 } });
|
||||
}
|
||||
}
|
||||
return read(file, options);
|
||||
});
|
||||
try { await reconcileRunCosts(db); } finally { spy.mockRestore(); }
|
||||
expect(lockWasAvailable).toBe(true);
|
||||
expect(reads.size).toBe(256);
|
||||
expect([...reads.values()]).toEqual(Array(256).fill(1));
|
||||
const charges = await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id));
|
||||
expect(charges).toHaveLength(8);
|
||||
expect(charges.find(charge => charge.heartbeatRunId === f.run.id)).toMatchObject({ inputTokens: 200, costCents: 20 });
|
||||
expect((await costService(db).summary(f.company.id)).spendCents).toBe(55);
|
||||
const listing = vi.spyOn(fs, "readdir");
|
||||
try {
|
||||
for (const run of runs) expect(await accountRunCost(db, run.id)).toBe(false);
|
||||
// An acknowledged run with a stale pending flag still needs its locked
|
||||
// cleanup, but neither form of duplicate finalization needs spool I/O.
|
||||
await db.update(heartbeatRuns).set({ costAccountingPending: true }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
expect(await accountRunCost(db, f.run.id)).toBe(false);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountingPending).toBe(false);
|
||||
expect(listing).not.toHaveBeenCalled();
|
||||
} finally { listing.mockRestore(); }
|
||||
await Promise.all([...foreignFiles].map(file => fs.rm(file)));
|
||||
});
|
||||
|
||||
it("reports over-limit advisory budgets without blocking, and tolerates legacy active zero limits", async () => {
|
||||
const f = await fixture(), budgets = budgetService(db);
|
||||
const policy = await budgets.upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 1, hardStopEnabled: false }, "board");
|
||||
await event(f, { costCents: "2.0000001" });
|
||||
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
|
||||
expect((await budgets.overview(f.company.id)).policies[0]).toMatchObject({ status: "hard_stop", observedAmountExact: "2.0000001", paused: false });
|
||||
// Older databases can contain an active zero policy; it must never become
|
||||
// an accidental hard stop during an upgrade or a projection inspection.
|
||||
await db.update(budgetPolicies).set({ amount: 0, isActive: true }).where(eq(budgetPolicies.id, policy.policyId));
|
||||
expect((await budgets.overview(f.company.id)).policies[0]).toMatchObject({ status: "ok", amount: 0 });
|
||||
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
|
||||
});
|
||||
it("recovers late bootstrap proof without replacing the stop owner's terminal metadata", async () => {
|
||||
const f = await fixture();
|
||||
await reserveRunBudget(db, f.company.id, f.run.id, null);
|
||||
const stopped = { executionCancellation: { state: "acknowledged" } };
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled", resultJson: stopped,
|
||||
usageJson: { accountingReceiptReady: false, accountingProviderWorkStarted: false },
|
||||
}).where(eq(heartbeatRuns.id, f.run.id));
|
||||
const integrity = accountingIntegrityService(db);
|
||||
expect((await integrity.health(f.company.id)).items[0].state).toBe("retryable");
|
||||
expect((await integrity.inspect(f.company.id)).findings).toEqual([]);
|
||||
await reconcileRunCosts(db);
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id));
|
||||
expect(run).toMatchObject({ status: "cancelled", resultJson: stopped, costAccountingPending: false });
|
||||
expect(run.costAccountedAt).not.toBeNull();
|
||||
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("released");
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toEqual([]);
|
||||
});
|
||||
it("rejects absent, foreign, finished and blocked dispatch without changing the ledger", async () => {
|
||||
const f = await fixture(), other = await fixture();
|
||||
await expect(reserveRunBudget(db,f.company.id,randomUUID(),null)).rejects.toThrow("not found");
|
||||
await expect(reserveRunBudget(db,other.company.id,f.run.id,null)).rejects.toThrow("not found");
|
||||
await expect(reserveRunBudget(db,f.company.id,f.run.id,other.project.id)).rejects.toThrow("not found");
|
||||
await db.update(heartbeatRuns).set({status:"failed"}).where(eq(heartbeatRuns.id,f.run.id));
|
||||
await expect(reserveRunBudget(db,f.company.id,f.run.id,null)).rejects.toThrow("no longer");
|
||||
await db.update(heartbeatRuns).set({status:"running",costAccountedAt:new Date()}).where(eq(heartbeatRuns.id,f.run.id));
|
||||
await expect(reserveRunBudget(db,f.company.id,f.run.id,null)).rejects.toThrow("no longer");
|
||||
await db.update(heartbeatRuns).set({costAccountedAt:null}).where(eq(heartbeatRuns.id,f.run.id));
|
||||
await budgetService(db).upsertPolicy(f.company.id,{scopeType:"agent",scopeId:f.agent.id,amount:1},"board");
|
||||
await event(f,{costCents:1});
|
||||
await expect(reserveRunBudget(db,f.company.id,f.run.id,null)).rejects.toThrow("paused");
|
||||
await expect(budgetService(db).upsertPolicy(f.company.id,{scopeType:"agent",scopeId:f.agent.id,amount:1,reservationCents:-1},"board")).rejects.toThrow("negative");
|
||||
});
|
||||
it("distinguishes legacy totals, missing acknowledgements, missing receipts, and corrupted receipt amounts", async () => {
|
||||
const f = await fixture(); const integrity = accountingIntegrityService(db);
|
||||
await db.insert(agentRuntimeState).values({companyId:f.company.id,agentId:f.agent.id,adapterType:"process",totalCostCents:4});
|
||||
expect((await integrity.inspect(f.company.id)).findings[0].kind).toBe("legacy_runtime");
|
||||
await db.update(heartbeatRuns).set({status:"failed",costAccountingPending:true,usageJson:{...usage,costUsdExact:"0.01",accountingReceiptReady:true}}).where(eq(heartbeatRuns.id,f.run.id));
|
||||
expect((await integrity.inspect(f.company.id)).findings.some(f => f.kind === "missing_acknowledgement")).toBe(true);
|
||||
expect(await integrity.retry(f.company.id,f.run.id,"board")).toEqual({accounted:true});
|
||||
await db.update(heartbeatRuns).set({usageJson:{...usage,costUsdExact:"0.02"}}).where(eq(heartbeatRuns.id,f.run.id));
|
||||
expect((await integrity.inspect(f.company.id)).findings.map(f=>f.kind)).toContain("receipt_mismatch");
|
||||
await db.update(heartbeatRuns).set({usageJson:{...usage,costUsdExact:"invalid"}}).where(eq(heartbeatRuns.id,f.run.id));
|
||||
expect((await integrity.inspect(f.company.id)).findings.find(f=>f.kind==="receipt_mismatch")?.expected.cents).toBe("invalid receipt");
|
||||
await db.delete(costEvents).where(eq(costEvents.heartbeatRunId,f.run.id));
|
||||
expect((await integrity.inspect(f.company.id)).findings.map(f=>f.kind)).toContain("missing_receipt");
|
||||
await expect(integrity.repair(f.company.id,"a".repeat(64)," ","board")).rejects.toThrow("reason");
|
||||
await expect(integrity.retry(f.company.id,randomUUID(),"board")).rejects.toThrow("not found");
|
||||
});
|
||||
it("records a failed operator retry and keeps it actionable", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(heartbeatRuns).set({status:"failed",costAccountingPending:true,usageJson:{inputTokens:0.1,accountingReceiptReady:true}}).where(eq(heartbeatRuns.id,f.run.id));
|
||||
const integrity = accountingIntegrityService(db);
|
||||
await expect(integrity.retry(f.company.id,f.run.id,"board")).rejects.toThrow("Invalid cost");
|
||||
const health = await integrity.health(f.company.id);
|
||||
expect(health.items[0]).toMatchObject({state:"retryable",attempts:1,lastError:"Invalid cost receipt"});
|
||||
expect(health.items[0].lastAttemptAt).not.toBeNull();
|
||||
});
|
||||
it("matches provider request IDs and run/model pairs, leaves ambiguous and non-inference lines unresolved", async () => {
|
||||
const f = await fixture(), service = billingReconciliationService(db);
|
||||
await event(f,{heartbeatRunId:f.run.id,providerRequestId:"request"}); await event(f,{heartbeatRunId:f.run.id,model:"second"});
|
||||
const line = {amountCents:"1",occurredAt:new Date().toISOString()};
|
||||
const invoice = await service.importInvoice(f.company.id,{biller:"fixture",externalId:"requests",currency:"USD",lines:[
|
||||
{...line,externalId:"provider",providerRequestId:"request"}, {...line,externalId:"model",runId:f.run.id,model:"second"},
|
||||
{...line,externalId:"ambiguous",runId:f.run.id}, {...line,externalId:"unknown",providerRequestId:"absent"},
|
||||
{...line,externalId:"unlinked"}, {...line,externalId:"fee",kind:"fee"}, {...line,externalId:"credit",kind:"credit"},
|
||||
]},"board");
|
||||
expect((await service.list(f.company.id))[0].id).toBe(invoice.id);
|
||||
const report = await service.reconcile(f.company.id,invoice.id);
|
||||
expect(Object.fromEntries(report.lines.map(l=>[l.externalId,l.status]))).toEqual({provider:"difference",model:"difference",ambiguous:"ambiguous",unknown:"unmatched",unlinked:"unmatched",fee:"non_inference",credit:"non_inference"});
|
||||
await expect(service.importInvoice(f.company.id,{biller:"fixture",externalId:"missing-run",currency:"USD",lines:[{...line,externalId:"run",runId:randomUUID()}]},"board")).rejects.toThrow("run not found");
|
||||
});
|
||||
it("rejects contradictory invoice evidence and reused correction keys, and preserves legacy originals", async () => {
|
||||
const f = await fixture(), service = billingReconciliationService(db); const charge = await event(f);
|
||||
await db.update(costEvents).set({reportedCostCents:null}).where(eq(costEvents.id,charge.id));
|
||||
const input = {idempotencyKey:"correction",expectedCents:"0.1234567",correctedCents:"1",reason:"Reviewed",pricing:{source:"operator" as const}};
|
||||
await expect(service.adjust(f.company.id,charge.id,{...input,invoiceLineId:randomUUID()},"board")).rejects.toThrow("not found");
|
||||
const invoice = await service.importInvoice(f.company.id,{biller:"fixture",externalId:"bad-evidence",currency:"USD",lines:[{externalId:"line",amountCents:"2",occurredAt:new Date().toISOString(),costEventId:charge.id}]},"board");
|
||||
const [line] = (await service.reconcile(f.company.id,invoice.id)).lines;
|
||||
await expect(service.adjust(f.company.id,charge.id,{...input,invoiceLineId:line.id},"board")).rejects.toThrow("does not support");
|
||||
const noMatch = await service.importInvoice(f.company.id,{biller:"fixture",externalId:"unverified",currency:"USD",lines:[{externalId:"line",amountCents:"1",occurredAt:new Date().toISOString()}]},"board");
|
||||
await expect(service.adjust(f.company.id,charge.id,{...input,invoiceLineId:(await service.reconcile(f.company.id,noMatch.id)).lines[0].id},"board")).rejects.toThrow("unverified");
|
||||
await service.adjust(f.company.id,charge.id,input,"board");
|
||||
await expect(service.adjust(f.company.id,charge.id,{...input,correctedCents:"2"},"board")).rejects.toThrow("different contents");
|
||||
expect((await db.select().from(costEvents).where(eq(costEvents.id,charge.id)))[0].reportedCostCents).toBe("0.1234567");
|
||||
});
|
||||
it("aggregates attempts exactly, rejects incomplete attempts, and never lets a late old attempt replace the current attempt", async () => {
|
||||
const f = await fixture(), spool = path.join(directory,randomUUID());
|
||||
const recorder = await createRunUsageRecorder(db,{companyId:f.company.id,runId:f.run.id,adapterType:"process"},spool);
|
||||
const first=randomUUID(), second=randomUUID();
|
||||
await recorder.capture({attemptId:first,usage,costUsdExact:"0.010000001",complete:true,billingType:"api"});
|
||||
await recorder.capture({attemptId:second,usage,costUsd:0.02,complete:false,billingType:"api"});
|
||||
await recorder.capture({attemptId:first,usage,costUsdExact:"0.010000001",complete:true,billingType:"api"});
|
||||
const result = await recorder.complete({exitCode:0,signal:null,timedOut:false,usage,costUsd:0.03,billingType:"api"});
|
||||
expect(result).toMatchObject({costUsdExact:"0.040000001",complete:true,usage:{inputTokens:14,cachedInputTokens:22,outputTokens:6}});
|
||||
await recorder.capture({attemptId:randomUUID(),usage,complete:false,billingType:"subscription"});
|
||||
const uncertain = await recorder.complete({exitCode:1,signal:null,timedOut:false,usage});
|
||||
expect(uncertain).toMatchObject({costUsd:0.040000001,costUsdExact:"0.040000001",costStatus:"unpriced",billingType:"unknown"});
|
||||
const partial = await recorder.complete({exitCode:1,signal:null,timedOut:false});
|
||||
expect(partial.complete).toBe(false);
|
||||
expect(partial.usage?.inputTokens).toBe(21);
|
||||
});
|
||||
it.each(["block", "allow"] as const)("keeps known attempted-run spend when unknown prices %s new work", async (unpricedUsagePolicy) => {
|
||||
const f = await fixture(), budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 100, unpricedUsagePolicy }, "board");
|
||||
await reserveRunBudget(db, f.company.id, f.run.id, null);
|
||||
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: f.run.id, adapterType: "process" }, path.join(directory, randomUUID()));
|
||||
await recorder.capture({ attemptId: randomUUID(), complete: true, billingType: "metered_api", usage, costUsdExact: "0.010000001" });
|
||||
const aggregate = await recorder.capture({ attemptId: randomUUID(), complete: true, billingType: "metered_api", usage, costUsd: null, costStatus: "unpriced" });
|
||||
expect(aggregate).toMatchObject({ costUsdExact: "0.010000001", costStatus: "unpriced", complete: true });
|
||||
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
expect(await accountRunCost(db, f.run.id)).toBe(true);
|
||||
expect(await accountRunCost(db, f.run.id)).toBe(false);
|
||||
expect(await costService(db).summary(f.company.id)).toMatchObject({ spendCentsExact: "1.0000001", unpricedEventCount: 1, pricingComplete: false, pendingRunCount: 0 });
|
||||
const [runtime] = await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id));
|
||||
expect(runtime.totalCostCents).toBe(1.0000001);
|
||||
const [reservation] = await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id));
|
||||
expect(reservation.state).toBe("settled");
|
||||
expect((await budgets.getInvocationBlock(f.company.id, f.agent.id)) !== null).toBe(unpricedUsagePolicy === "block");
|
||||
});
|
||||
|
||||
it("refuses invalid checkpoints and does not certify a run after capture failure", async () => {
|
||||
const f = await fixture();
|
||||
const recorder = await createRunUsageRecorder(db,{companyId:f.company.id,runId:f.run.id,adapterType:"process"},path.join(directory,randomUUID()));
|
||||
await expect(recorder.capture({complete:true,costUsdExact:"NaN"})).rejects.toThrow();
|
||||
const result = await recorder.complete({exitCode:0,signal:null,timedOut:false,usage,costUsd:0.01});
|
||||
expect(result.complete).toBe(false);
|
||||
await expect(createRunUsageRecorder(db,{companyId:f.company.id,runId:randomUUID(),adapterType:"process"},path.join(directory,randomUUID()))).rejects.toThrow("cannot accept");
|
||||
});
|
||||
it("preserves disk errors, rejects missing runs, and handles invalid spool entries without deleting evidence", async () => {
|
||||
const f=await fixture(), spool=path.join(directory,randomUUID());
|
||||
const envelope:UsageReceiptEnvelope={schema:"paperclip/accounting-receipt/v1",id:randomUUID(),companyId:f.company.id,runId:randomUUID(),sourceId:randomUUID(),sequence:1,receivedAt:new Date().toISOString(),adapterType:"process",receipt:{complete:true,costUsd:1}};
|
||||
await expect(persistUsageReceipt(db,envelope)).rejects.toThrow("not found");
|
||||
const rename=vi.spyOn(fs,"rename").mockRejectedValueOnce(new Error("Disk rename unavailable"));
|
||||
try { await expect(spoolUsageReceipt(envelope,spool)).rejects.toThrow("Disk rename"); } finally { rename.mockRestore(); }
|
||||
expect(await fs.readdir(spool)).toEqual([]);
|
||||
await fs.writeFile(path.join(spool,"large.json"),"x".repeat(1024*1024+1));
|
||||
expect((await replayUsageReceipts(db,spool)).failed).toBe(1);
|
||||
const read=vi.spyOn(fs,"readdir").mockRejectedValueOnce(Object.assign(new Error("denied"),{code:"EACCES"}));
|
||||
try { await expect(replayUsageReceipts(db,spool)).rejects.toThrow("denied"); } finally { read.mockRestore(); }
|
||||
});
|
||||
it("uses the isolated instance spool and retains evidence when retry rotation itself fails", async () => {
|
||||
const f = await fixture();
|
||||
const envelope: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", id: randomUUID(), companyId: f.company.id, runId: f.run.id,
|
||||
sourceId: randomUUID(), sequence: 1, receivedAt: new Date().toISOString(), adapterType: "process", receipt: { complete: true, costUsd: 0 } };
|
||||
const file = await spoolUsageReceipt(envelope);
|
||||
expect(path.dirname(file)).toBe(usageReceiptSpoolPath());
|
||||
expect((await replayUsageReceipts(db)).replayed).toBe(1);
|
||||
const spool = path.join(directory, randomUUID()); await fs.mkdir(spool);
|
||||
const bad = path.join(spool, "bad.json"); await fs.writeFile(bad, "invalid");
|
||||
const rename = vi.spyOn(fs, "rename").mockRejectedValueOnce(new Error("read-only directory"));
|
||||
try { expect(await replayUsageReceipts(db, spool)).toEqual({ replayed: 0, failed: 1 }); } finally { rename.mockRestore(); }
|
||||
expect(await fs.readFile(bad, "utf8")).toBe("invalid");
|
||||
});
|
||||
|
||||
});
|
||||
@@ -0,0 +1,502 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { promises as fs } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { agents, agentRuntimeState, budgetReservations, companies, costEvents, costAdjustments, createDb, heartbeatRuns, runUsageReceipts, issues, nativeRunFinalizations } from "@paperclipai/db";
|
||||
import { accountingIntegrityService } from "../services/accounting-integrity.js";
|
||||
import { billingReconciliationService } from "../services/billing-reconciliation.js";
|
||||
import { budgetService } from "../services/budgets.js";
|
||||
import { reserveRunBudget } from "../services/budget-reservations.js";
|
||||
import { costService } from "../services/costs.js";
|
||||
import { accountRunCost, reconcileRunCosts } from "../services/run-cost-accounting.js";
|
||||
import { createRunUsageRecorder, persistUsageReceipt, replayUsageReceipts, spoolUsageReceipt, type UsageReceiptEnvelope } from "../services/usage-receipts.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { createUsageCheckpointLog } from "@paperclipai/adapter-utils/usage-checkpoint";
|
||||
import { runChildProcess } from "@paperclipai/adapter-utils/server-utils";
|
||||
import { createClaudeStreamParser, parseClaudeStreamJson } from "../../../packages/adapters/claude-local/src/server/parse.js";
|
||||
|
||||
import { parseCodexJsonl } from "../../../packages/adapters/codex-local/src/server/parse.js";
|
||||
import { parseOpenCodeJsonl } from "../../../packages/adapters/opencode-local/src/server/parse.js";
|
||||
import { parseCursorJsonl } from "../../../packages/adapters/cursor-local/src/server/parse.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("accounting operations and durable receipts (PostgreSQL)", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let directory: string;
|
||||
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("paperclip-accounting-operators-"); db = createDb(database.connectionString); directory = await fs.mkdtemp(path.join(os.tmpdir(), "accounting-spool-test-")); }, 30_000);
|
||||
afterAll(async () => { await database?.cleanup(); await fs.rm(directory, { recursive: true, force: true }); });
|
||||
async function fixture() {
|
||||
const [company] = await db.insert(companies).values({ name: "Accounting", issuePrefix: `A${randomUUID().slice(0,7)}` }).returning();
|
||||
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process", status: "idle" }).returning();
|
||||
return { company, agent };
|
||||
}
|
||||
async function runFor(f: Awaited<ReturnType<typeof fixture>>, status = "running") {
|
||||
return (await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, invocationSource: "on_demand", status }).returning())[0];
|
||||
}
|
||||
async function receipt(f: Awaited<ReturnType<typeof fixture>>, costCents: number | string = "1.1234567") {
|
||||
return costService(db).createEvent(f.company.id, { agentId: f.agent.id, provider: "test", model: "test-model", costCents, occurredAt: new Date(), idempotencyKey: randomUUID() });
|
||||
}
|
||||
|
||||
it("keeps the last nanocent exact in storage, reports, projections and admission", async () => {
|
||||
const f = await fixture();
|
||||
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 2_000_000_000 }, "board");
|
||||
await receipt(f, "1999999999.9999999");
|
||||
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
|
||||
expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("1999999999.9999999");
|
||||
await receipt(f, "0.0000001");
|
||||
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).not.toBeNull();
|
||||
const huge = await fixture();
|
||||
await receipt(huge, "99999999999999999.0000001");
|
||||
expect((await accountingIntegrityService(db).inspect(huge.company.id)).findings).toEqual([]);
|
||||
});
|
||||
|
||||
it("admits only affordable concurrent reservations and settles atomically with the receipt", async () => {
|
||||
const f = await fixture(); const runs = await Promise.all(Array.from({ length: 4 }, () => runFor(f)));
|
||||
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 10, reservationCents: "6" }, "board");
|
||||
const results = await Promise.allSettled(runs.map(run => reserveRunBudget(db, f.company.id, run.id, null)));
|
||||
expect(results.filter(r => r.status === "fulfilled")).toHaveLength(1);
|
||||
const index = results.findIndex(r => r.status === "fulfilled"); const run = runs[index];
|
||||
expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("6.0000000");
|
||||
await expect(reserveRunBudget(db, f.company.id, run.id, null)).rejects.toThrow("already reserved");
|
||||
// Reservations never disappear merely because wall-clock time passed.
|
||||
await db.update(budgetReservations).set({ createdAt: new Date("2000-01-01") }).where(eq(budgetReservations.runId, run.id));
|
||||
const denied = runs[(index + 1) % runs.length];
|
||||
await expect(reserveRunBudget(db, f.company.id, denied.id, null)).rejects.toThrow("reserved");
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date(), usageJson: { costUsdExact: "0.04", billingType: "metered_api", accountingReceiptReady: true } }).where(eq(heartbeatRuns.id, run.id));
|
||||
expect(await accountRunCost(db, run.id)).toBe(true);
|
||||
expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("0.0000000");
|
||||
await reserveRunBudget(db, f.company.id, denied.id, null);
|
||||
expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]);
|
||||
});
|
||||
|
||||
it("reuses a held native reservation only for its current recovery lease", async () => {
|
||||
const f = await fixture(); const run = await runFor(f);
|
||||
const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Recover" }).returning();
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, run.id));
|
||||
await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: run.id, issueId: issue.id,
|
||||
phase: "running", leaseOwner: "successor", leaseExpiresAt: new Date(Date.now() + 60_000) });
|
||||
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 10, reservationCents: "6" }, "board");
|
||||
const original = await reserveRunBudget(db, f.company.id, run.id, null);
|
||||
await expect(reserveRunBudget(db, f.company.id, run.id, null)).rejects.toThrow("already reserved");
|
||||
await expect(reserveRunBudget(db, f.company.id, run.id, null, {}, "stale-owner")).rejects.toThrow("already reserved");
|
||||
expect(await reserveRunBudget(db, f.company.id, run.id, null, {}, "successor")).toEqual({ ...original, reused: true });
|
||||
expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("6.0000000");
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(0) }).where(eq(nativeRunFinalizations.runId, run.id));
|
||||
await expect(reserveRunBudget(db, f.company.id, run.id, null, {}, "successor")).rejects.toThrow("already reserved");
|
||||
});
|
||||
|
||||
it.each([false, true])("settles absent usage without hiding explicit unknown pricing (%s)", async (unpriced) => {
|
||||
const f = await fixture(); const run = await runFor(f);
|
||||
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100, reservationCents: "10" }, "board");
|
||||
await reserveRunBudget(db, f.company.id, run.id, null);
|
||||
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "process" }, directory);
|
||||
await recorder.complete({ exitCode: 0, signal: null, timedOut: false, ...(unpriced ? { provider: "moonshot", costStatus: "unpriced" as const, costUsd: null } : {}) });
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
|
||||
expect(await accountRunCost(db, run.id)).toBe(true);
|
||||
expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("0.0000000");
|
||||
const summary = await costService(db).summary(f.company.id);
|
||||
expect(summary.unpricedEventCount).toBe(unpriced ? 1 : 0);
|
||||
const block = await budgetService(db).getInvocationBlock(f.company.id, f.agent.id);
|
||||
if (unpriced) expect(block).not.toBeNull(); else expect(block).toBeNull();
|
||||
});
|
||||
|
||||
it.each([
|
||||
...["turn.failed", "error", "turn.completed"].map(type => ({
|
||||
label: `Codex ${type}`, adapterType: "codex_local", parse: () => parseCodexJsonl(JSON.stringify({ type })),
|
||||
})),
|
||||
...["result", "step_finish"].map(type => ({
|
||||
label: `Cursor ${type}`, adapterType: "cursor", parse: () => parseCursorJsonl(JSON.stringify({ type })),
|
||||
})),
|
||||
{ label: "OpenCode step_finish", adapterType: "opencode_local", parse: () => parseOpenCodeJsonl(JSON.stringify({ type: "step_finish" })) },
|
||||
])("keeps $label without usage unpriced and its reservation held", async ({ adapterType, parse }) => {
|
||||
const f = await fixture(); const run = await runFor(f);
|
||||
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100, reservationCents: "10" }, "board");
|
||||
await reserveRunBudget(db, f.company.id, run.id, null);
|
||||
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType }, directory);
|
||||
const parsed = parse();
|
||||
const accounting = { provider: "openai", biller: "openai", billingType: "api" as const, model: "gpt-6-astra", usageBasis: "per_run" as const,
|
||||
usage: parsed.usageReported ? parsed.usage : undefined, costUsd: null, costStatus: "unpriced" as const };
|
||||
await recorder.capture({ ...accounting, complete: parsed.usageComplete });
|
||||
await recorder.complete({ ...accounting, exitCode: 1, signal: null, timedOut: false, usageComplete: parsed.usageComplete });
|
||||
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
|
||||
const [stored] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));
|
||||
expect(stored.usageJson).toMatchObject({ costUsd: null, costUsdExact: null, costStatus: "unpriced", accountingReceiptReady: false });
|
||||
expect(await accountRunCost(db, run.id)).toBe(false);
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toEqual([]);
|
||||
expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("10.0000000");
|
||||
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).not.toBeNull();
|
||||
});
|
||||
|
||||
it("recovers a flushed zero-usage OpenCode receipt after stopping before the adapter result is saved", async () => {
|
||||
const f = await fixture(); const run = await runFor(f);
|
||||
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100, reservationCents: "10" }, "board");
|
||||
await reserveRunBudget(db, f.company.id, run.id, null);
|
||||
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "opencode_local" }, directory);
|
||||
const log = createUsageCheckpointLog(async () => {}, async receipt => { await recorder.capture(receipt); }, records => ({
|
||||
provider: "openai", biller: "openai", billingType: "unknown", model: "openai/test", usageBasis: "per_run",
|
||||
usage: parseOpenCodeJsonl(records).usage, costUsd: null, complete: false,
|
||||
}));
|
||||
await log("stdout", JSON.stringify({ type: "step_finish", part: { tokens: { input: 0, output: 0 } } }));
|
||||
await log.flush({ complete: true });
|
||||
// No recorder.complete(adapterResult): the process stopped during cleanup.
|
||||
await db.update(heartbeatRuns).set({ status: "interrupted", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
|
||||
expect(await accountRunCost(db, run.id)).toBe(true);
|
||||
const events = await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id));
|
||||
expect(events).toHaveLength(1);
|
||||
expect(events[0]).toMatchObject({ inputTokens: 0, outputTokens: 0, costCents: 0, costStatus: "unpriced" });
|
||||
expect((await accountingIntegrityService(db).health(f.company.id))).toMatchObject({ heldReservationCents: "0.0000000", pendingRunCount: 0 });
|
||||
expect(await accountRunCost(db, run.id)).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps an interrupted partial receipt pending and settles its reservation when evidence becomes complete", async () => {
|
||||
const f = await fixture(); const run = await runFor(f);
|
||||
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100, reservationCents: "10" }, "board");
|
||||
await reserveRunBudget(db, f.company.id, run.id, null);
|
||||
const usage = { accountingReceiptReady: false, costUsdExact: "0.01", inputTokens: 5, provider: "test", billingType: "metered_api" };
|
||||
await db.update(heartbeatRuns).set({ status: "interrupted", usageJson: usage }).where(eq(heartbeatRuns.id, run.id));
|
||||
const integrity = accountingIntegrityService(db);
|
||||
expect((await integrity.health(f.company.id)).pendingRunCount).toBe(1);
|
||||
expect((await integrity.health(f.company.id)).items[0]).toMatchObject({ runId: run.id, state: "waiting_for_receipt" });
|
||||
expect((await costService(db).summary(f.company.id)).pendingRunCount).toBe(1);
|
||||
expect((await integrity.inspect(f.company.id)).findings.some(f => f.kind === "missing_acknowledgement")).toBe(true);
|
||||
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).not.toBeNull();
|
||||
expect(await accountRunCost(db, run.id)).toBe(false);
|
||||
expect((await integrity.health(f.company.id)).heldReservationCents).toBe("10.0000000");
|
||||
await db.update(heartbeatRuns).set({ usageJson: { ...usage, accountingReceiptReady: true } }).where(eq(heartbeatRuns.id, run.id));
|
||||
await reconcileRunCosts(db);
|
||||
expect((await costService(db).summary(f.company.id))).toMatchObject({ pendingRunCount: 0, spendCents: 1 });
|
||||
expect((await integrity.health(f.company.id)).heldReservationCents).toBe("0.0000000");
|
||||
expect((await integrity.inspect(f.company.id)).findings).toEqual([]);
|
||||
expect(await accountRunCost(db, run.id)).toBe(false);
|
||||
});
|
||||
|
||||
it("independently detects and repairs corrupted projections with a stale-review guard and audit", async () => {
|
||||
const f = await fixture(); const run = await runFor(f, "succeeded");
|
||||
await db.update(heartbeatRuns).set({ costAccountingPending: true, usageJson: { accountingReceiptReady: true, costUsdExact: "0.012345678", inputTokens: 7, cachedInputTokens: 3, outputTokens: 11 } }).where(eq(heartbeatRuns.id, run.id));
|
||||
await accountRunCost(db, run.id);
|
||||
const service = accountingIntegrityService(db);
|
||||
expect((await service.inspect(f.company.id)).findings).toEqual([]);
|
||||
await db.update(companies).set({ spentMonthlyCents: 999 }).where(eq(companies.id, f.company.id));
|
||||
await db.update(agents).set({ spentMonthlyCents: 888 }).where(eq(agents.id, f.agent.id));
|
||||
await db.update(agentRuntimeState).set({ totalCostCents: 777, totalInputTokens: 600 }).where(eq(agentRuntimeState.agentId, f.agent.id));
|
||||
const review = await service.inspect(f.company.id);
|
||||
expect(review.findings.map(f => f.kind)).toEqual(["agent_projection", "company_projection", "runtime_projection"]);
|
||||
await db.update(companies).set({ spentMonthlyCents: 998 }).where(eq(companies.id, f.company.id));
|
||||
await expect(service.repair(f.company.id, review.fingerprint, "Fix drift", "board")).rejects.toThrow("changed since inspection");
|
||||
const fresh = await service.inspect(f.company.id);
|
||||
expect((await service.repair(f.company.id, fresh.fingerprint, "Fix drift", "board")).findings).toEqual([]);
|
||||
const audit = await db.execute(sql`select details from activity_log where company_id = ${f.company.id} and action = 'accounting.projections_repaired'`);
|
||||
expect(audit).toHaveLength(1);
|
||||
expect(await accountRunCost(db, run.id)).toBe(false);
|
||||
expect((await service.inspect(f.company.id)).findings).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(["before", "after", "concurrent"])("accounts separate run charges %s the provider receipt without false integrity findings", async (timing) => {
|
||||
const f = await fixture(); const run = await runFor(f, "succeeded");
|
||||
await db.update(heartbeatRuns).set({ costAccountingPending: true, usageJson: { costUsdExact: "0.01", inputTokens: 7, cachedInputTokens: 3, outputTokens: 11 } }).where(eq(heartbeatRuns.id, run.id));
|
||||
const extra = { agentId: f.agent.id, heartbeatRunId: run.id, provider: "test", model: "tool", costCents: "0.1234567", inputTokens: 2, cachedInputTokens: 4, outputTokens: 6, occurredAt: new Date(), idempotencyKey: "extra-charge" };
|
||||
const costs = costService(db);
|
||||
if (timing === "before") await costs.createEvent(f.company.id, extra);
|
||||
if (timing === "concurrent") await Promise.all([costs.createEvent(f.company.id, extra), accountRunCost(db, run.id)]);
|
||||
else await accountRunCost(db, run.id);
|
||||
const event = await costs.createEvent(f.company.id, extra);
|
||||
// Replaying either source cannot count it twice.
|
||||
await costs.createEvent(f.company.id, extra);
|
||||
expect(await accountRunCost(db, run.id)).toBe(false);
|
||||
const [runtime] = await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id));
|
||||
expect(runtime).toMatchObject({ totalCostCents: 1.1234567, totalInputTokens: 9, totalCachedInputTokens: 7, totalOutputTokens: 17 });
|
||||
const integrity = accountingIntegrityService(db);
|
||||
expect((await integrity.inspect(f.company.id)).findings).toEqual([]);
|
||||
await billingReconciliationService(db).adjust(f.company.id, event.id, { idempotencyKey: "extra-correction", expectedCents: "0.1234567", correctedCents: "0.2234567", reason: "Provider correction", pricing: { source: "provider_invoice", evidence: "external charge" } }, "board");
|
||||
expect((await costs.summary(f.company.id)).spendCentsExact).toBe("1.2234567");
|
||||
expect((await integrity.inspect(f.company.id)).findings).toEqual([]);
|
||||
await db.update(agentRuntimeState).set({ totalCostCents: 999 }).where(eq(agentRuntimeState.agentId, f.agent.id));
|
||||
const review = await integrity.inspect(f.company.id);
|
||||
expect(review.findings.map(f => f.kind)).toEqual(["runtime_projection"]);
|
||||
expect((await integrity.repair(f.company.id, review.fingerprint, "Fix drift", "board")).findings).toEqual([]);
|
||||
// An extra charge cannot conceal a missing original provider receipt.
|
||||
await db.delete(costEvents).where(and(eq(costEvents.heartbeatRunId, run.id), eq(costEvents.idempotencyKey, `heartbeat:${run.id}:final`)));
|
||||
expect((await integrity.inspect(f.company.id)).findings.some(f => f.kind === "missing_receipt")).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects public charges that impersonate the provider receipt namespace", async () => {
|
||||
const f = await fixture();
|
||||
await expect(costService(db).createEvent(f.company.id, { agentId: f.agent.id, provider: "test", model: "test", costCents: 1, occurredAt: new Date(), idempotencyKey: "heartbeat:fake:final" })).rejects.toThrow("reserved");
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(["zero", "paid", "tokens"] as const)("preserves model attribution after a retry with %s earlier usage", async kind => {
|
||||
const f = await fixture(); const run = await runFor(f);
|
||||
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, directory);
|
||||
await recorder.capture({ attemptId: randomUUID(), provider: "anthropic", billingType: "metered_api",
|
||||
usage: { inputTokens: kind === "tokens" ? 1 : 0, outputTokens: 0, cacheWriteTokens: undefined },
|
||||
costUsd: kind === "paid" ? 0.001 : 0, complete: true });
|
||||
const success = { attemptId: randomUUID(), provider: "anthropic", biller: "anthropic", billingType: "metered_api" as const, model: "mixed",
|
||||
usage: { inputTokens: 30, outputTokens: 4, cachedInputTokens: 100 }, costUsd: 0.007, complete: true,
|
||||
usageByModel: [
|
||||
{ model: "large", costUsd: 0.005, usage: { inputTokens: 20, outputTokens: 3, cachedInputTokens: 100 } },
|
||||
{ model: "small", costUsd: 0.002, usage: { inputTokens: 10, outputTokens: 1 } },
|
||||
] };
|
||||
await recorder.capture(success);
|
||||
const final = await recorder.complete({ ...success, exitCode: 0, signal: null, timedOut: false, usageComplete: true });
|
||||
expect(final.usageByModel).toEqual(kind === "zero" ? success.usageByModel : undefined);
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
|
||||
expect(await accountRunCost(db, run.id)).toBe(true);
|
||||
const rows = await costService(db).byProvider(f.company.id);
|
||||
expect(rows.map(row => row.model).sort()).toEqual(kind === "zero" ? ["large", "small"] : ["mixed"]);
|
||||
expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe(kind === "paid" ? "0.8000000" : "0.7000000");
|
||||
});
|
||||
|
||||
it("imports invoices idempotently and appends reviewed corrections without losing the provider receipt", async () => {
|
||||
const f = await fixture();
|
||||
const originalPricing = { source: "rate_card" as const, version: "2026-09-01", inputCentsPerMillion: "12.3456789" };
|
||||
const event = await costService(db).createEvent(f.company.id, { agentId: f.agent.id, provider: "test", model: "test-model",
|
||||
costCents: "1.1234567", occurredAt: new Date(), pricingProvenance: originalPricing });
|
||||
const service = billingReconciliationService(db);
|
||||
const invoice = { biller: "test", externalId: "invoice-1", currency: "usd", lines: [{ externalId: "request-1", amountCents: "2.0000001", occurredAt: new Date().toISOString(), costEventId: event.id }] };
|
||||
const imported = await service.importInvoice(f.company.id, invoice, "board");
|
||||
expect((await service.importInvoice(f.company.id, invoice, "board")).id).toBe(imported.id);
|
||||
await expect(service.importInvoice(f.company.id, { ...invoice, currency: "EUR" }, "board")).rejects.toThrow("different contents");
|
||||
const review = await service.reconcile(f.company.id, imported.id);
|
||||
expect(review.lines[0]).toMatchObject({ status: "difference", recordedCents: "1.1234567", differenceCents: "0.8765434" });
|
||||
const input = { idempotencyKey: "fix-1", expectedCents: "1.1234567", correctedCents: "2.0000001", invoiceLineId: review.lines[0].id, reason: "Provider invoice", pricing: { source: "provider_invoice" as const, evidence: "invoice-1" } };
|
||||
const corrected = await service.adjust(f.company.id, event.id, input, "board");
|
||||
expect(corrected.previousPricing).toEqual(originalPricing);
|
||||
expect(corrected.pricing).toEqual(input.pricing);
|
||||
expect((await service.adjust(f.company.id, event.id, input, "board")).id).toBe(corrected.id);
|
||||
expect((await service.reconcile(f.company.id, imported.id)).lines[0].status).toBe("matched");
|
||||
const [row] = await db.select().from(costEvents).where(eq(costEvents.id, event.id));
|
||||
expect(row.reportedCostCents).toBe("1.1234567"); expect(row.receiptHash).toBe(event.receiptHash);
|
||||
expect((await service.adjustments(f.company.id, event.id))).toHaveLength(1);
|
||||
expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]);
|
||||
await expect(service.adjust(f.company.id, event.id, { ...input, idempotencyKey: "stale" }, "board")).rejects.toThrow("changed since review");
|
||||
const second = await service.adjust(f.company.id, event.id, { ...input, invoiceLineId: undefined, idempotencyKey: "fix-2",
|
||||
expectedCents: "2.0000001", correctedCents: "3", pricing: { source: "operator" }, reason: "Reviewed updated evidence" }, "board");
|
||||
expect(second.previousPricing).toEqual(input.pricing);
|
||||
expect((await service.adjustments(f.company.id, event.id))).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("refuses cross-company references, guesses, duplicate invoice matches, and currency conversion", async () => {
|
||||
const f = await fixture(), foreign = await fixture(); const event = await receipt(f);
|
||||
const service = billingReconciliationService(db);
|
||||
const base = { biller: "test", externalId: "invoice", currency: "USD", lines: [{ externalId: "line", amountCents: "1", occurredAt: new Date().toISOString(), costEventId: event.id }] };
|
||||
await expect(service.importInvoice(foreign.company.id, base, "board")).rejects.toThrow("not found");
|
||||
await expect(service.adjust(foreign.company.id, event.id, { idempotencyKey: "bad", expectedCents: 0, correctedCents: 1, reason: "test", pricing: { source: "operator" } }, "board")).rejects.toThrow("not found");
|
||||
const duplicate = await service.importInvoice(f.company.id, { ...base, lines: [...base.lines, { ...base.lines[0], externalId: "second" }] }, "board");
|
||||
expect((await service.reconcile(f.company.id, duplicate.id)).lines.map(l => l.status)).toEqual(["ambiguous", "ambiguous"]);
|
||||
const eur = await service.importInvoice(f.company.id, { ...base, externalId: "eur", currency: "EUR" }, "board");
|
||||
expect((await service.reconcile(f.company.id, eur.id)).lines[0].status).toBe("unsupported_currency");
|
||||
await expect(service.reconcile(foreign.company.id, eur.id)).rejects.toThrow("not found");
|
||||
expect(await db.select().from(costAdjustments).where(eq(costAdjustments.companyId, f.company.id))).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(["provider_invoice", "rate_card"] as const)("prices unknown usage from %s and preserves its estimate status", async (source) => {
|
||||
const f = await fixture(); const run = await runFor(f, "failed");
|
||||
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 10 }, "board");
|
||||
await db.update(heartbeatRuns).set({ costAccountingPending: true, usageJson: { accountingReceiptReady: true, inputTokens: 9, provider: "test" } }).where(eq(heartbeatRuns.id, run.id));
|
||||
await accountRunCost(db, run.id);
|
||||
expect((await accountingIntegrityService(db).health(f.company.id)).unpricedEventCount).toBe(1);
|
||||
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).not.toBeNull();
|
||||
const [event] = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, run.id));
|
||||
await billingReconciliationService(db).adjust(f.company.id, event.id, { idempotencyKey: "pricing", expectedCents: 0, correctedCents: "1.0000001", reason: "Verified provider invoice", pricing: { source, evidence: "billing-export" } }, "board");
|
||||
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
|
||||
expect((await accountingIntegrityService(db).health(f.company.id)).unpricedEventCount).toBe(0);
|
||||
expect((await costService(db).summary(f.company.id)).estimatedEventCount).toBe(source === "rate_card" ? 1 : 0);
|
||||
expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]);
|
||||
});
|
||||
|
||||
it("spools a receipt during database failure, replays once, fences old sources, and preserves partial uncertainty", async () => {
|
||||
const f = await fixture(); const run = await runFor(f); const spool = path.join(directory, randomUUID());
|
||||
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "process" }, spool);
|
||||
const transaction = vi.spyOn(db, "transaction").mockRejectedValueOnce(new Error("Database unavailable"));
|
||||
await recorder.capture({ costUsdExact: "0.010000001", usage: { inputTokens: 7, outputTokens: 2 }, complete: false });
|
||||
transaction.mockRestore();
|
||||
expect((await fs.readdir(spool)).filter(n => n.endsWith(".json"))).toHaveLength(1);
|
||||
expect(await replayUsageReceipts(db, spool)).toEqual({ replayed: 1, failed: 0 });
|
||||
expect(await replayUsageReceipts(db, spool)).toEqual({ replayed: 0, failed: 0 });
|
||||
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, run.id));
|
||||
expect(await accountRunCost(db, run.id)).toBe(false);
|
||||
expect((await accountingIntegrityService(db).health(f.company.id)).items[0].state).toBe("waiting_for_receipt");
|
||||
await recorder.complete({ exitCode: 0, signal: null, timedOut: false, costUsdExact: "0.010000001", usage: { inputTokens: 7, outputTokens: 2 } });
|
||||
expect(await accountRunCost(db, run.id)).toBe(true);
|
||||
const receipts = await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, run.id));
|
||||
expect(receipts).toHaveLength(2);
|
||||
await persistUsageReceipt(db, receipts[0].receiptJson as UsageReceiptEnvelope);
|
||||
expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("1.0000001");
|
||||
const tampered = { ...(receipts[0].receiptJson as UsageReceiptEnvelope), receipt: { complete: true, costUsd: 123 } };
|
||||
await expect(persistUsageReceipt(db, tampered)).rejects.toThrow("Conflicting usage");
|
||||
});
|
||||
|
||||
it("settles a complete streamed Claude receipt when the final stdout tail clips its result", async () => {
|
||||
const f = await fixture(); const run = await runFor(f);
|
||||
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 100, reservationCents: 10 }, "board");
|
||||
await reserveRunBudget(db, f.company.id, run.id, null);
|
||||
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, directory);
|
||||
const consume = createClaudeStreamParser();
|
||||
const log = createUsageCheckpointLog(async () => {}, async receipt => { await recorder.capture(receipt); }, records => {
|
||||
const parsed = consume(records);
|
||||
return { provider: "anthropic", biller: "anthropic", model: "claude-fable-5-1", billingType: "metered_api",
|
||||
usage: parsed.usage ?? undefined, costUsd: parsed.costUsd, complete: parsed.resultJson !== null };
|
||||
});
|
||||
const proc = await runChildProcess(run.id, process.execPath, ["-e", `console.log(JSON.stringify({
|
||||
type: "result", subtype: "success", result: "x".repeat(5 * 1024 * 1024),
|
||||
total_cost_usd: 0.012345678, usage: { input_tokens: 7, cache_read_input_tokens: 3, output_tokens: 2 }
|
||||
}))`], { cwd: directory, env: {}, timeoutSec: 10, graceSec: 1, onLog: log });
|
||||
await log.flush();
|
||||
const tail = parseClaudeStreamJson(proc.stdout);
|
||||
expect(tail.resultJson).toBeNull();
|
||||
const final = await recorder.complete({ exitCode: proc.exitCode, signal: proc.signal, timedOut: proc.timedOut,
|
||||
provider: "anthropic", billingType: "metered_api", usage: tail.usage ?? undefined, costUsd: tail.costUsd,
|
||||
usageComplete: tail.resultJson !== null });
|
||||
expect(final).toMatchObject({ complete: true, costUsd: 0.012345678, usage: { inputTokens: 7, cachedInputTokens: 3, outputTokens: 2 } });
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
|
||||
expect(await accountRunCost(db, run.id)).toBe(true);
|
||||
expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("1.2345678");
|
||||
expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("0.0000000");
|
||||
});
|
||||
|
||||
it("holds failed retry accounting when flush throws before final completion", async () => {
|
||||
const f = await fixture(); const run = await runFor(f); const spool = path.join(directory, randomUUID());
|
||||
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 100, reservationCents: 10 }, "board");
|
||||
await reserveRunBudget(db, f.company.id, run.id, null);
|
||||
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, spool);
|
||||
const attempt = () => {
|
||||
const consume = createClaudeStreamParser();
|
||||
return createUsageCheckpointLog(async () => {}, async receipt => { await recorder.capture(receipt); }, records => {
|
||||
const parsed = consume(records);
|
||||
return { provider: "anthropic", billingType: "metered_api", usage: parsed.usage ?? undefined,
|
||||
costUsd: parsed.costUsd, complete: parsed.resultJson !== null };
|
||||
});
|
||||
};
|
||||
const rejectedResume = attempt();
|
||||
await rejectedResume("stdout", JSON.stringify({ type: "result", subtype: "error_during_execution", total_cost_usd: 0,
|
||||
usage: { input_tokens: 0, output_tokens: 0 } }) + "\n");
|
||||
await rejectedResume.flush();
|
||||
const [prior] = await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, run.id));
|
||||
expect((prior.receiptJson as UsageReceiptEnvelope).receipt.complete).toBe(true);
|
||||
const paidRetry = attempt();
|
||||
const write = vi.spyOn(fs, "rename").mockRejectedValueOnce(new Error("Receipt disk full"));
|
||||
try {
|
||||
await paidRetry("stdout", JSON.stringify({ type: "result", subtype: "success", total_cost_usd: 0.25,
|
||||
usage: { input_tokens: 10, output_tokens: 2 } }) + "\n");
|
||||
await expect(paidRetry.flush()).rejects.toThrow("Receipt disk full");
|
||||
} finally { write.mockRestore(); }
|
||||
// This is the adapter's throw path: complete() is never called.
|
||||
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
|
||||
const read = async () => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0];
|
||||
expect(await read()).toMatchObject({ costAccountingPending: true, costAccountedAt: null,
|
||||
accountingLastError: "usage_capture_failed", usageJson: { accountingReceiptReady: false, accountingCaptureFailed: true } });
|
||||
expect(await accountRunCost(db, run.id)).toBe(false);
|
||||
// Old and later complete envelopes must not clear lost-capture evidence.
|
||||
await spoolUsageReceipt(prior.receiptJson as UsageReceiptEnvelope, spool);
|
||||
await spoolUsageReceipt({ ...prior.receiptJson as UsageReceiptEnvelope, id: randomUUID(), sequence: 100 }, spool);
|
||||
expect(await replayUsageReceipts(db, spool)).toEqual({ replayed: 2, failed: 0 });
|
||||
expect((await read()).usageJson?.accountingReceiptReady).toBe(false);
|
||||
const replacement = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, spool);
|
||||
expect((await replacement.complete({ exitCode: 0, signal: null, timedOut: false, costUsd: 0.25 })).complete).toBe(false);
|
||||
// Even a finalizer overwriting readiness or claiming pre-provider failure cannot erase the fence.
|
||||
await db.update(heartbeatRuns).set({ usageJson: { ...(await read()).usageJson, accountingReceiptReady: true },
|
||||
resultJson: { executionRecovery: { providerWorkStarted: false } } }).where(eq(heartbeatRuns.id, run.id));
|
||||
expect(await accountRunCost(db, run.id)).toBe(false);
|
||||
expect((await read()).costAccountedAt).toBeNull();
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, run.id))).toEqual([]);
|
||||
expect(await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id))).toEqual([]);
|
||||
expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("10.0000000");
|
||||
});
|
||||
|
||||
it.each(["capture", "complete", "invalid"] as const)("persists the failure fence before %s rejects", async kind => {
|
||||
const f = await fixture(); const run = await runFor(f);
|
||||
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "process" }, directory);
|
||||
await recorder.persistFailure(); // Healthy failure finalization has no fence to save.
|
||||
await recorder.capture({ costUsd: 0, complete: true });
|
||||
const rename = vi.spyOn(fs, "rename");
|
||||
if (kind !== "invalid") rename.mockRejectedValueOnce(new Error("Receipt disk full"));
|
||||
try {
|
||||
const result = kind === "complete"
|
||||
? recorder.complete({ exitCode: 0, signal: null, timedOut: false, costUsd: 1 })
|
||||
: recorder.capture({ costUsd: kind === "invalid" ? -1 : 1, complete: true });
|
||||
await expect(result).rejects.toThrow();
|
||||
const [stored] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));
|
||||
expect(stored.usageJson).toMatchObject({ accountingCaptureFailed: true, accountingReceiptReady: false });
|
||||
} finally { rename.mockRestore(); }
|
||||
});
|
||||
|
||||
it("retries a failed durable fence before allowing failure finalization", async () => {
|
||||
const f = await fixture(); const run = await runFor(f);
|
||||
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "process" }, directory);
|
||||
await recorder.capture({ costUsd: 0, complete: true });
|
||||
const transaction = vi.spyOn(db, "transaction").mockRejectedValueOnce(new Error("Database unavailable"));
|
||||
try {
|
||||
await expect(recorder.capture({ costUsd: -1, complete: true })).rejects.toThrow();
|
||||
expect(transaction).toHaveBeenCalledTimes(1);
|
||||
transaction.mockRejectedValueOnce(new Error("Database still unavailable"));
|
||||
await expect(recorder.persistFailure()).rejects.toThrow("Database still unavailable");
|
||||
await recorder.persistFailure();
|
||||
} finally { transaction.mockRestore(); }
|
||||
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, run.id));
|
||||
expect(await accountRunCost(db, run.id)).toBe(false);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toMatchObject({
|
||||
costAccountedAt: null, accountingLastError: "usage_capture_failed",
|
||||
usageJson: { accountingCaptureFailed: true, accountingReceiptReady: false },
|
||||
});
|
||||
});
|
||||
|
||||
it("does not let a replaced recorder's failed capture poison its successor", async () => {
|
||||
const f = await fixture(); const run = await runFor(f);
|
||||
const input = { companyId: f.company.id, runId: run.id, adapterType: "process" };
|
||||
const old = await createRunUsageRecorder(db, input, directory);
|
||||
const current = await createRunUsageRecorder(db, input, directory);
|
||||
await current.capture({ costUsd: 0.25, complete: true });
|
||||
await expect(old.capture({ costUsd: -1, complete: true })).rejects.toThrow();
|
||||
await old.persistFailure();
|
||||
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, run.id));
|
||||
expect(await accountRunCost(db, run.id)).toBe(true);
|
||||
expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("25.0000000");
|
||||
});
|
||||
|
||||
it.each(["more_tokens", "more_cost", "new_attempt"])("keeps additional incomplete provider work pending (%s)", async (kind) => {
|
||||
const f = await fixture(); const run = await runFor(f);
|
||||
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, directory);
|
||||
await recorder.capture({ provider: "anthropic", usage: { inputTokens: 7, outputTokens: 2 }, costUsd: 0.01, complete: true });
|
||||
if (kind === "new_attempt") await recorder.capture({ attemptId: randomUUID(), provider: "anthropic", usage: { inputTokens: 1, outputTokens: 0 }, complete: false });
|
||||
const final = await recorder.complete({ exitCode: 1, signal: null, timedOut: false, usageComplete: false,
|
||||
usage: { inputTokens: kind === "more_tokens" ? 8 : 7, outputTokens: 2 }, costUsd: kind === "more_cost" ? 0.02 : 0.01 });
|
||||
expect(final.complete).toBe(false);
|
||||
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
|
||||
expect(await accountRunCost(db, run.id)).toBe(false);
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")("rejects recorder setup before dispatch when its new directory parent cannot be flushed", async () => {
|
||||
const f = await fixture(); const run = await runFor(f);
|
||||
const spool = path.join(directory, randomUUID(), "receipts");
|
||||
const realParent = await fs.realpath(directory);
|
||||
const open = fs.open.bind(fs);
|
||||
const probe = vi.spyOn(fs, "open").mockImplementation(async (...args: Parameters<typeof fs.open>) => {
|
||||
if (args[0] === realParent) throw Object.assign(new Error("unflushed parent"), { code: "EACCES" });
|
||||
return open(...args);
|
||||
});
|
||||
try {
|
||||
await expect(createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, spool)).rejects.toMatchObject({ code: "EACCES" });
|
||||
expect(probe.mock.calls.some(([file]) => String(file).endsWith(".probe"))).toBe(false);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].usageJson?.accountingReceiptSourceId).toBeUndefined();
|
||||
} finally { probe.mockRestore(); }
|
||||
await expect(createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, spool)).resolves.toBeDefined();
|
||||
});
|
||||
|
||||
it("retains invalid spool evidence and bounds recovery work", async () => {
|
||||
const spool = path.join(directory, randomUUID()); await fs.mkdir(spool);
|
||||
await fs.writeFile(path.join(spool, "bad.json"), "not json");
|
||||
expect(await replayUsageReceipts(db, spool, 1)).toEqual({ replayed: 0, failed: 1 });
|
||||
expect((await fs.readdir(spool))[0]).toMatch(/^retry-/);
|
||||
expect(await replayUsageReceipts(db, path.join(spool, "missing"))).toEqual({ replayed: 0, failed: 0 });
|
||||
const f = await fixture(); const run = await runFor(f);
|
||||
const unknown: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", id: randomUUID(), companyId: f.company.id, runId: run.id,
|
||||
sourceId: randomUUID(), sequence: 1, receivedAt: new Date().toISOString(), adapterType: "process", receipt: { complete: true, costUsd: 10 } };
|
||||
await spoolUsageReceipt(unknown, spool);
|
||||
expect((await replayUsageReceipts(db, spool, 1)).replayed).toBe(1);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].costAccountingPending).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,266 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { promises as fs } from "node:fs";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import { agents, budgetPolicies, companies, costEvents, createDb, heartbeatRuns, issues, nativeRunFinalizations, projects, type Db } from "@paperclipai/db";
|
||||
import { billingReconciliationService } from "../services/billing-reconciliation.js";
|
||||
import { accountingIntegrityService } from "../services/accounting-integrity.js";
|
||||
import { budgetService, budgetServiceInTransaction } from "../services/budgets.js";
|
||||
import { withAccountingTransaction } from "../services/accounting-transaction.js";
|
||||
import { costService } from "../services/costs.js";
|
||||
import { createCostAccountingReconciler } from "../services/run-cost-accounting.js";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
function deferred() {
|
||||
let resolve!: () => void;
|
||||
const promise = new Promise<void>(done => { resolve = done; });
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
describe("accounting performance invariants (PostgreSQL)", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>, db: Db;
|
||||
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("accounting-performance-"); db = createDb(database.connectionString); }, 30_000);
|
||||
afterEach(() => { vi.restoreAllMocks(); vi.useRealTimers(); });
|
||||
afterAll(async () => { await database?.cleanup(); });
|
||||
async function fixture() {
|
||||
const [company] = await db.insert(companies).values({ name: "Performance", issuePrefix: `P${randomUUID().slice(0, 7)}` }).returning();
|
||||
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process" }).returning();
|
||||
const [project] = await db.insert(projects).values({ companyId: company.id, name: "Work" }).returning();
|
||||
return { company, agent, project };
|
||||
}
|
||||
function trace() {
|
||||
const queries: string[] = [];
|
||||
const traced = drizzle(db.$client, { logger: { logQuery(query) { queries.push(query); } } }) as unknown as Db;
|
||||
return { traced, ledgerReads: () => queries.filter(query => query.startsWith("select") && query.includes('from "cost_events"')) };
|
||||
}
|
||||
|
||||
it("reads health while a writer holds the company lock and keeps counts/items in one snapshot", async () => {
|
||||
const f = await fixture();
|
||||
const writer = await db.$client.reserve();
|
||||
await writer`begin`;
|
||||
try {
|
||||
await writer`select id from companies where id = ${f.company.id} for no key update`;
|
||||
const original = db.transaction.bind(db);
|
||||
const transaction = vi.spyOn(db, "transaction").mockImplementation((callback, config) => original(async tx => {
|
||||
// A lock attempt fails promptly. There is no elapsed-time pass/fail assertion.
|
||||
await tx.execute(sql`set local lock_timeout = '200ms'`);
|
||||
const execute = tx.execute.bind(tx);
|
||||
let counted = false;
|
||||
vi.spyOn(tx, "execute").mockImplementation(query => {
|
||||
const result = execute(query);
|
||||
const executeRaw = result.execute.bind(result);
|
||||
vi.spyOn(result, "execute").mockImplementation(async () => {
|
||||
const rows = await executeRaw();
|
||||
if (!counted) {
|
||||
counted = true;
|
||||
// Commit a pending row between health's counts and item list.
|
||||
await writer`insert into heartbeat_runs (company_id,agent_id,status,cost_accounting_pending)
|
||||
values (${f.company.id},${f.agent.id},'failed',true)`;
|
||||
await writer`commit`;
|
||||
}
|
||||
return rows;
|
||||
});
|
||||
return result;
|
||||
});
|
||||
return callback(tx);
|
||||
}, config));
|
||||
expect(await accountingIntegrityService(db).health(f.company.id)).toMatchObject({ pendingRunCount: 0, items: [] });
|
||||
expect(transaction).toHaveBeenCalledWith(expect.any(Function), { isolationLevel: "repeatable read", accessMode: "read only" });
|
||||
transaction.mockRestore();
|
||||
expect(await accountingIntegrityService(db).health(f.company.id)).toMatchObject({ pendingRunCount: 1, items: [expect.objectContaining({ state: "retryable" })] });
|
||||
} finally { await writer`rollback`; writer.release(); }
|
||||
// Remove the fixture's pending debt before the global recovery tests.
|
||||
await db.execute(sql`delete from heartbeat_runs where company_id = ${f.company.id}`);
|
||||
await expect(accountingIntegrityService(db).health(randomUUID())).rejects.toThrow("Company not found");
|
||||
});
|
||||
|
||||
it("keeps summary incompleteness consistent when an unpriced run settles between reads", async () => {
|
||||
const f = await fixture();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id,
|
||||
status: "failed", costAccountingPending: true, finishedAt: new Date() }).returning();
|
||||
const writer = await db.$client.reserve();
|
||||
await writer`begin`;
|
||||
try {
|
||||
await writer`select id from companies where id = ${f.company.id} for no key update`;
|
||||
const original = db.transaction.bind(db);
|
||||
const transaction = vi.spyOn(db, "transaction").mockImplementation((callback, config) => original(async tx => {
|
||||
await tx.execute(sql`set local lock_timeout = '200ms'`);
|
||||
const select = tx.select.bind(tx);
|
||||
vi.spyOn(tx, "select").mockImplementation((...args: Parameters<typeof tx.select>) => {
|
||||
const query = select(...args);
|
||||
const from = query.from.bind(query);
|
||||
vi.spyOn(query, "from").mockImplementation(table => {
|
||||
const result = from(table);
|
||||
if (table === costEvents) {
|
||||
const execute = result.execute.bind(result);
|
||||
vi.spyOn(result, "execute").mockImplementation(async () => {
|
||||
const rows = await execute();
|
||||
await writer`insert into cost_events (company_id,agent_id,provider,model,cost_cents,cost_status,billing_type,occurred_at)
|
||||
values (${f.company.id},${f.agent.id},'fixture','fixture',0,'unpriced','metered_api',now())`;
|
||||
await writer`update heartbeat_runs set cost_accounting_pending=false where id=${run.id}`;
|
||||
await writer`commit`;
|
||||
return rows;
|
||||
});
|
||||
}
|
||||
return result;
|
||||
});
|
||||
return query;
|
||||
});
|
||||
return callback(tx);
|
||||
}, config));
|
||||
expect(await costService(db).summary(f.company.id)).toMatchObject({ pendingRunCount: 1, unpricedEventCount: 0, pricingComplete: false });
|
||||
transaction.mockRestore();
|
||||
expect(await costService(db).summary(f.company.id)).toMatchObject({ pendingRunCount: 0, unpricedEventCount: 1, pricingComplete: false });
|
||||
} finally { await writer`rollback`; writer.release(); }
|
||||
});
|
||||
|
||||
it("shares receipt replay across overlapping ticks and releases the guard after replay failure", async () => {
|
||||
const reconcile = createCostAccountingReconciler(db);
|
||||
const gate = deferred();
|
||||
const read = vi.spyOn(fs, "readdir").mockImplementationOnce(async () => { await gate.promise; throw new Error("Spool unavailable"); });
|
||||
const first = reconcile(), second = reconcile();
|
||||
expect(second).toBe(first);
|
||||
expect(read).toHaveBeenCalledTimes(1);
|
||||
const settled = Promise.allSettled([first, second]);
|
||||
gate.resolve();
|
||||
expect(await settled).toEqual([expect.objectContaining({ status: "rejected" }), expect.objectContaining({ status: "rejected" })]);
|
||||
read.mockRestore();
|
||||
expect(await reconcile()).toMatchObject({ scanned: 0, accounted: 0 });
|
||||
});
|
||||
|
||||
it("keeps the guard through budget delivery and retries after reconciliation failure", async () => {
|
||||
const f = await fixture();
|
||||
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 1 }, "board");
|
||||
await costService(db).createEvent(f.company.id, { agentId: f.agent.id, provider: "fixture", model: "fixture", costCents: 1, occurredAt: new Date() });
|
||||
const entered = deferred(), gate = deferred();
|
||||
const cancelWorkForScope = vi.fn(async () => { entered.resolve(); await gate.promise; });
|
||||
const reconcile = createCostAccountingReconciler(db, { cancelWorkForScope });
|
||||
const first = reconcile();
|
||||
try {
|
||||
await entered.promise;
|
||||
expect(reconcile()).toBe(first);
|
||||
expect(cancelWorkForScope).toHaveBeenCalledTimes(1);
|
||||
} finally { gate.resolve(); }
|
||||
await first;
|
||||
const select = vi.spyOn(db, "select").mockImplementationOnce(() => { throw new Error("Database unavailable"); });
|
||||
await expect(reconcile()).rejects.toThrow("Database unavailable");
|
||||
select.mockRestore();
|
||||
const next = reconcile(); expect(next).not.toBe(first);
|
||||
await next;
|
||||
});
|
||||
|
||||
it("uses one ledger observation per overview policy and per admission policy", async () => {
|
||||
const f = await fixture();
|
||||
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 100 }, "board");
|
||||
const { traced, ledgerReads } = trace();
|
||||
expect((await budgetService(traced).overview(f.company.id)).policies[0]).toMatchObject({ observedAmountExact: "0.0000000", status: "ok" });
|
||||
expect(ledgerReads()).toHaveLength(1);
|
||||
await budgetService(traced).getInvocationBlock(f.company.id, f.agent.id);
|
||||
expect(ledgerReads()).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("shares a scan across mixed policy scopes/windows without trusting monthly projections", async () => {
|
||||
const f = await fixture(), foreign = await fixture();
|
||||
const [other] = await db.insert(agents).values({ companyId: f.company.id, name: "Other", role: "engineer", adapterType: "process" }).returning();
|
||||
const values = [
|
||||
{ scopeType: "company", scopeId: f.company.id, windowKind: "calendar_month_utc", amount: 4 },
|
||||
{ scopeType: "agent", scopeId: f.agent.id, windowKind: "lifetime", amount: 10 },
|
||||
{ scopeType: "project", scopeId: f.project.id, windowKind: "lifetime", amount: 9 },
|
||||
{ scopeType: "agent", scopeId: other.id, windowKind: "calendar_month_utc", amount: 1 },
|
||||
];
|
||||
await db.insert(budgetPolicies).values(values.map(value => ({ ...value, companyId: f.company.id })));
|
||||
const base = { provider: "fixture", model: "fixture", costCents: 3, occurredAt: new Date() };
|
||||
const [event] = await db.insert(costEvents).values([
|
||||
{ ...base, companyId: f.company.id, agentId: f.agent.id, projectId: f.project.id, costCents: 7, occurredAt: new Date("2000-01-01") },
|
||||
{ ...base, companyId: f.company.id, agentId: f.agent.id, projectId: f.project.id },
|
||||
{ ...base, companyId: f.company.id, agentId: other.id },
|
||||
{ ...base, companyId: foreign.company.id, agentId: foreign.agent.id, costCents: 999 },
|
||||
]).returning();
|
||||
await db.update(companies).set({ spentMonthlyCents: 0 }).where(eq(companies.id, f.company.id));
|
||||
const { traced, ledgerReads } = trace();
|
||||
await withAccountingTransaction(traced, f.company.id, tx => budgetServiceInTransaction(tx).evaluateCostEvent(event));
|
||||
expect(ledgerReads()).toHaveLength(1);
|
||||
const incidents = await db.execute<{ scope_type: string; amount_observed: string }>(sql`select scope_type,amount_observed::text from budget_incidents where company_id = ${f.company.id} order by scope_type`);
|
||||
expect(incidents).toEqual([
|
||||
{ scope_type: "agent", amount_observed: "10.0000000" },
|
||||
{ scope_type: "company", amount_observed: "6.0000000" },
|
||||
{ scope_type: "project", amount_observed: "10.0000000" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps unpriced and pending-run guards scoped when batching native recovery observations", async () => {
|
||||
const f = await fixture();
|
||||
const [other] = await db.insert(agents).values({ companyId: f.company.id, name: "Other", role: "engineer", adapterType: "process" }).returning();
|
||||
await db.insert(budgetPolicies).values([
|
||||
{ companyId: f.company.id, scopeType: "company", scopeId: f.company.id, windowKind: "calendar_month_utc", amount: 100 },
|
||||
{ companyId: f.company.id, scopeType: "agent", scopeId: f.agent.id, windowKind: "lifetime", amount: 100 },
|
||||
{ companyId: f.company.id, scopeType: "project", scopeId: f.project.id, windowKind: "lifetime", amount: 100 },
|
||||
]);
|
||||
const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Recover" }).returning();
|
||||
const [native] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id,
|
||||
status: "failed", runtimeMode: "native", nativeIssueId: issue.id, costAccountingPending: true, finishedAt: new Date() }).returning();
|
||||
await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: native.id, issueId: issue.id, phase: "retryable_failure" });
|
||||
const base = { companyId: f.company.id, provider: "fixture", model: "fixture", costCents: 0, occurredAt: new Date() };
|
||||
const [event] = await db.insert(costEvents).values([
|
||||
{ ...base, agentId: f.agent.id, projectId: f.project.id },
|
||||
{ ...base, agentId: other.id, projectId: f.project.id, occurredAt: new Date("2000-01-01"), costStatus: "unpriced" },
|
||||
]).returning();
|
||||
await withAccountingTransaction(db, f.company.id, tx => budgetServiceInTransaction(tx).evaluateCostEvent(event));
|
||||
const overview = await budgetService(db).overview(f.company.id);
|
||||
expect(overview.activeIncidents.map(row => row.scopeType)).toEqual(["project"]);
|
||||
expect(overview.policies).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ scopeType: "company", status: "ok", pendingRunCount: 1, unpricedEventCount: 0 }),
|
||||
expect.objectContaining({ scopeType: "agent", status: "ok", pendingRunCount: 1, unpricedEventCount: 0 }),
|
||||
expect.objectContaining({ scopeType: "project", status: "hard_stop", pendingRunCount: 0, unpricedEventCount: 1 }),
|
||||
]));
|
||||
await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "failed",
|
||||
costAccountingPending: true, finishedAt: new Date(), usageJson: { ledgerScope: { projectId: f.project.id } } });
|
||||
await withAccountingTransaction(db, f.company.id, tx => budgetServiceInTransaction(tx).evaluateCostEvent(event));
|
||||
expect((await budgetService(db).overview(f.company.id)).activeIncidents.map(row => row.scopeType).sort()).toEqual(["agent", "company", "project"]);
|
||||
});
|
||||
|
||||
|
||||
it("keeps incident amounts and overview windows tied to the observed UTC month across midnight", async () => {
|
||||
const f = await fixture();
|
||||
await db.insert(budgetPolicies).values({ companyId: f.company.id, scopeType: "company", scopeId: f.company.id,
|
||||
windowKind: "calendar_month_utc", amount: 1 });
|
||||
const [event] = await db.insert(costEvents).values({ companyId: f.company.id, agentId: f.agent.id,
|
||||
provider: "fixture", model: "fixture", costCents: 1, occurredAt: new Date("2026-01-31T12:00:00Z") }).returning();
|
||||
const traced = drizzle(db.$client, { logger: { logQuery(query) {
|
||||
if (query.startsWith("select") && query.includes('from "cost_events"')) vi.setSystemTime(new Date("2026-02-01T00:00:00Z"));
|
||||
} } }) as unknown as Db;
|
||||
vi.useFakeTimers({ toFake: ["Date"] });
|
||||
vi.setSystemTime(new Date("2026-01-31T23:59:59.999Z"));
|
||||
await withAccountingTransaction(traced, f.company.id, tx => budgetServiceInTransaction(tx).evaluateCostEvent(event));
|
||||
const incident = (await budgetService(db).overview(f.company.id)).activeIncidents[0];
|
||||
expect(incident).toMatchObject({ amountObserved: 1, windowStart: new Date("2026-01-01T00:00:00Z"), windowEnd: new Date("2026-02-01T00:00:00Z") });
|
||||
vi.setSystemTime(new Date("2026-01-31T23:59:59.999Z"));
|
||||
expect((await budgetService(traced).overview(f.company.id)).policies[0]).toMatchObject({
|
||||
observedAmountExact: "1.0000000", windowStart: new Date("2026-01-01T00:00:00Z"), windowEnd: new Date("2026-02-01T00:00:00Z"),
|
||||
});
|
||||
// The next operation uses February and releases the prior month's pause.
|
||||
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
|
||||
});
|
||||
|
||||
|
||||
it.each(["inspection", "invoice comparison"])("does not take the company write lock during %s", async kind => {
|
||||
const f = await fixture();
|
||||
const invoice = await billingReconciliationService(db).importInvoice(f.company.id, { biller: "fixture", externalId: "invoice", currency: "USD",
|
||||
lines: [{ externalId: "line", amountCents: "1", occurredAt: new Date().toISOString() }] }, "board");
|
||||
const writer = await db.$client.reserve();
|
||||
await writer`begin`;
|
||||
try {
|
||||
await writer`select id from companies where id = ${f.company.id} for no key update`;
|
||||
const original = db.transaction.bind(db);
|
||||
const transaction = vi.spyOn(db, "transaction").mockImplementation((callback, config) => original(async tx => {
|
||||
await tx.execute(sql`set local lock_timeout = '200ms'`);
|
||||
return callback(tx);
|
||||
}, config));
|
||||
if (kind === "inspection") expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]);
|
||||
else expect((await billingReconciliationService(db).reconcile(f.company.id, invoice.id)).lines).toHaveLength(1);
|
||||
expect(transaction).toHaveBeenCalledWith(expect.any(Function), { isolationLevel: "repeatable read", accessMode: "read only" });
|
||||
} finally { await writer`rollback`; writer.release(); }
|
||||
});
|
||||
|
||||
});
|
||||
@@ -0,0 +1,110 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { createConnection, createServer, type Socket } from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { agents, companies, costEvents, createDb, heartbeatRuns } from "@paperclipai/db";
|
||||
import { costService } from "../services/costs.js";
|
||||
import { accountingIntegrityService } from "../services/accounting-integrity.js";
|
||||
import { replayUsageReceipts } from "../services/usage-receipts.js";
|
||||
import { accountRunCost } from "../services/run-cost-accounting.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
const root = fileURLToPath(new URL("../../..", import.meta.url));
|
||||
(support.supported && process.platform !== "win32" ? describe : describe.skip)("accounting across process and connection failures", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("accounting-processes-"); db = createDb(database.connectionString); }, 30_000);
|
||||
afterAll(async () => { await database?.cleanup(); });
|
||||
async function fixture() {
|
||||
const [company] = await db.insert(companies).values({ name: "Process", issuePrefix: `P${randomUUID().slice(0,7)}` }).returning();
|
||||
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process", status: "idle" }).returning();
|
||||
return { company, agent };
|
||||
}
|
||||
function worker(mode: string, input: unknown) {
|
||||
return spawn(process.execPath, ["cli/node_modules/tsx/dist/cli.mjs", "server/src/__tests__/fixtures/accounting-writer.ts", mode, JSON.stringify(input)], {
|
||||
cwd: root, detached: true, stdio: ["ignore", "pipe", "pipe"],
|
||||
env: { PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: process.env.TMPDIR, PAPERCLIP_ACCOUNTING_TEST_DATABASE: database.connectionString, PAPERCLIP_TELEMETRY_DISABLED: "1" },
|
||||
});
|
||||
}
|
||||
it("deduplicates the same receipt from independent server processes", async () => {
|
||||
const f = await fixture();
|
||||
const receipt = { agentId: f.agent.id, provider: "fixture", model: "fixture", costCents: "0.1234567", idempotencyKey: "shared-key", occurredAt: new Date().toISOString() };
|
||||
const children = Array.from({ length: 4 }, () => worker("write", { companyId: f.company.id, receipt }));
|
||||
try {
|
||||
await Promise.all(children.map(child => new Promise<void>((resolve, reject) => {
|
||||
let log = ""; child.stdout.on("data", d => log += d); child.stderr.on("data", d => log += d);
|
||||
child.once("error", reject); child.once("exit", code => code === 0 && log.includes("WRITES_COMMITTED") ? resolve() : reject(new Error(log)));
|
||||
})));
|
||||
expect((await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id)))).toHaveLength(1);
|
||||
expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]);
|
||||
} finally { for (const child of children) if (child.exitCode === null && child.signalCode === null) process.kill(-child.pid!, "SIGKILL"); }
|
||||
}, 45_000);
|
||||
|
||||
it("recovers a receipt after SIGKILL before its first database persistence", async () => {
|
||||
const f = await fixture(); const directory = await mkdtemp(path.join(tmpdir(), "accounting-early-kill-"));
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "running", costAccountingPending: true, usageJson: { accountingReceiptReady: false } }).returning();
|
||||
const child = worker("receipt", { companyId: f.company.id, runId: run.id, directory });
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
let log = ""; child.stdout.on("data", data => { log += data; if (log.includes("RECEIPT_DURABLE")) resolve(); });
|
||||
child.stderr.on("data", data => log += data); child.once("error", reject); child.once("exit", () => reject(new Error(`Exited before checkpoint: ${log}`)));
|
||||
});
|
||||
const stopped = new Promise(resolve => child.once("exit", resolve)); process.kill(-child.pid!, "SIGKILL"); await stopped;
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toEqual([]);
|
||||
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
|
||||
expect(await replayUsageReceipts(db, directory)).toEqual({ replayed: 1, failed: 0 });
|
||||
expect(await accountRunCost(db, run.id)).toBe(true);
|
||||
expect(await accountRunCost(db, run.id)).toBe(false);
|
||||
expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("1.2345678");
|
||||
expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]);
|
||||
} finally {
|
||||
if (child.exitCode === null && child.signalCode === null) process.kill(-child.pid!, "SIGKILL");
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
}, 45_000);
|
||||
|
||||
it("retries safely when PostgreSQL commits but its acknowledgement is lost", async () => {
|
||||
const f = await fixture(); const target = new URL(database.connectionString); const sockets = new Set<Socket>(); let dropped = false;
|
||||
// Parse PostgreSQL server frames, suppress precisely CommandComplete(COMMIT),
|
||||
// then break TCP. The database has committed while the writer sees failure.
|
||||
const proxy = createServer(client => {
|
||||
const upstream = createConnection({ host: target.hostname, port: Number(target.port) }); sockets.add(client); sockets.add(upstream);
|
||||
client.on("error", () => {}); upstream.on("error", () => client.destroy()); client.pipe(upstream);
|
||||
let pending = Buffer.alloc(0);
|
||||
upstream.on("data", chunk => {
|
||||
pending = Buffer.concat([pending, chunk]);
|
||||
while (pending.length >= 5) {
|
||||
const size = pending.readInt32BE(1) + 1;
|
||||
if (size < 5 || size > 16 * 1024 * 1024) { client.destroy(); upstream.destroy(); return; }
|
||||
if (pending.length < size) return;
|
||||
const message = pending.subarray(0, size); pending = pending.subarray(size);
|
||||
if (!dropped && message[0] === 67 && message.subarray(5).toString() === "COMMIT\0") {
|
||||
dropped = true; client.destroy(); upstream.destroy(); return;
|
||||
}
|
||||
client.write(message);
|
||||
}
|
||||
});
|
||||
client.on("close", () => { sockets.delete(client); upstream.destroy(); });
|
||||
upstream.on("close", () => { sockets.delete(upstream); client.destroy(); });
|
||||
});
|
||||
await new Promise<void>(resolve => proxy.listen(0, "127.0.0.1", resolve));
|
||||
const forwarded = new URL(database.connectionString); forwarded.hostname = "127.0.0.1"; forwarded.port = String((proxy.address() as { port: number }).port); forwarded.searchParams.set("sslmode", "disable");
|
||||
const writer = createDb(forwarded.toString());
|
||||
const receipt = { agentId: f.agent.id, provider: "fixture", model: "fixture", costCents: "0.9999999", idempotencyKey: "ambiguous-commit", occurredAt: new Date() };
|
||||
try {
|
||||
await expect(costService(writer).createEvent(f.company.id, receipt)).rejects.toThrow();
|
||||
expect(dropped).toBe(true);
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(1);
|
||||
await costService(writer).createEvent(f.company.id, receipt);
|
||||
expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("0.9999999");
|
||||
expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]);
|
||||
} finally {
|
||||
await writer.$client.end({ timeout: 1 }); for (const socket of sockets) socket.destroy(); await new Promise<void>(resolve => proxy.close(() => resolve()));
|
||||
}
|
||||
}, 30_000);
|
||||
});
|
||||
@@ -0,0 +1,741 @@
|
||||
import { upsertBudgetPolicySchema } from "@paperclipai/shared";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { activityLog, agentRuntimeState, agentWakeupRequests, agents, approvals, budgetIncidents, budgetPolicies, companies, completionContracts, costEvents, createDb, financeEvents, goals, heartbeatRuns, issues, nativeRunResults, projects } from "@paperclipai/db";
|
||||
import { reserveRunBudget } from "../services/budget-reservations.js";
|
||||
import { createRunDispatch } from "../modules/run-dispatch/index.js";
|
||||
import { costService, createCostEventInTransaction } from "../services/costs.js";
|
||||
import { budgetService, withCurrentBudgetEnforcement, type BudgetEnforcementScope } from "../services/budgets.js";
|
||||
import { accountRunCost, reconcileRunCosts } from "../services/run-cost-accounting.js";
|
||||
import { financeService } from "../services/finance.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { hoistModuleGraph } from "./helpers/hoist-module-graph.js";
|
||||
import { NativeRunCoordinatorStore } from "../services/native-runtime/native-run-coordinator-store.js";
|
||||
import { CONTROL_PLANE_CONFORMANCE_RESULT, CONTROL_PLANE_CONFORMANCE_TERMINAL } from "../vendor/paperclip-runner/testing.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
const databaseDescribe = support.supported ? describe : describe.skip;
|
||||
|
||||
databaseDescribe("cost accounting reliability (PostgreSQL)", () => {
|
||||
const services = hoistModuleGraph(() => {}, async () => {
|
||||
const [agentModule, companyModule, approvalModule] = await Promise.all([
|
||||
import("../services/agents.js"), import("../services/companies.js"), import("../services/approvals.js"),
|
||||
]);
|
||||
return { agentService: agentModule.agentService, companyService: companyModule.companyService, approvalService: approvalModule.approvalService };
|
||||
});
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("paperclip-cost-reliability-");
|
||||
db = createDb(database.connectionString);
|
||||
}, 30_000);
|
||||
afterAll(async () => { await database?.cleanup(); });
|
||||
|
||||
async function fixture() {
|
||||
const [company] = await db.insert(companies).values({ name: "Accounting", issuePrefix: `T${randomUUID().slice(0, 7)}` }).returning();
|
||||
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", status: "active", adapterType: "codex_local" }).returning();
|
||||
const [project] = await db.insert(projects).values({ companyId: company.id, name: "Project", status: "in_progress" }).returning();
|
||||
const [issue] = await db.insert(issues).values({ companyId: company.id, projectId: project.id, title: "Work" }).returning();
|
||||
const [goal] = await db.insert(goals).values({ companyId: company.id, title: "Goal" }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, invocationSource: "on_demand" }).returning();
|
||||
const receipt = { agentId: agent.id, provider: "test", model: "test", billingType: "metered_api", costCents: 100, occurredAt: new Date() };
|
||||
return { company, agent, project, issue, goal, run, receipt };
|
||||
}
|
||||
|
||||
it.each([false, true])("deletes linked accounting records before company runs (concurrent writer: %s)", async (concurrentWriter) => {
|
||||
const f = await fixture();
|
||||
const other = await fixture();
|
||||
await costService(db).createEvent(other.company.id, other.receipt);
|
||||
const charge = await costService(db).createEvent(f.company.id, { ...f.receipt, heartbeatRunId: f.run.id, issueId: f.issue.id });
|
||||
await financeService(db).createEvent(f.company.id, {
|
||||
eventKind: "platform_fee", direction: "debit", biller: "paperclip", amountCents: 5, currency: "USD", occurredAt: new Date(),
|
||||
heartbeatRunId: f.run.id, costEventId: charge.id,
|
||||
});
|
||||
const [policy] = await db.insert(budgetPolicies).values({ companyId: f.company.id, scopeType: "company", scopeId: f.company.id,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc", amount: 1000 }).returning();
|
||||
await db.insert(budgetIncidents).values({ companyId: f.company.id, policyId: policy.id, scopeType: "company", scopeId: f.company.id,
|
||||
metric: policy.metric, windowKind: policy.windowKind, windowStart: new Date(), windowEnd: new Date(),
|
||||
thresholdType: "warning", amountLimit: 1000, amountObserved: 100 });
|
||||
let removing: Promise<{ value?: unknown; error?: unknown }> | undefined;
|
||||
try {
|
||||
if (concurrentWriter) {
|
||||
await db.transaction(async tx => {
|
||||
await tx.select().from(companies).where(eq(companies.id, f.company.id)).for("no key update");
|
||||
removing = services.value.companyService(db).remove(f.company.id).then(value => ({ value }), error => ({ error }));
|
||||
await vi.waitFor(async () => {
|
||||
const waiting = await db.execute(sql`select pid from pg_stat_activity where datname = current_database()
|
||||
and wait_event_type = 'Lock' and query like '%companies%for no key update%'`);
|
||||
expect(waiting.length).toBeGreaterThan(0);
|
||||
}, { timeout: 3000 });
|
||||
await createCostEventInTransaction(tx as unknown as ReturnType<typeof createDb>, f.company.id, { ...f.receipt, heartbeatRunId: f.run.id }, []);
|
||||
});
|
||||
expect(await removing!).toMatchObject({ value: { id: f.company.id } });
|
||||
} else expect(await services.value.companyService(db).remove(f.company.id)).toMatchObject({ id: f.company.id });
|
||||
expect(await db.select().from(companies).where(eq(companies.id, f.company.id))).toHaveLength(0);
|
||||
for (const table of [heartbeatRuns, costEvents, financeEvents, budgetPolicies, budgetIncidents]) {
|
||||
expect(await db.select().from(table).where(eq(table.companyId, f.company.id))).toHaveLength(0);
|
||||
}
|
||||
expect((await costService(db).summary(other.company.id)).spendCents).toBe(100);
|
||||
expect(await services.value.companyService(db).remove(f.company.id)).toBeNull();
|
||||
} finally { await removing; }
|
||||
});
|
||||
|
||||
it("serializes agent deletion before child locks and preserves a concurrent settled charge", async () => {
|
||||
const f = await fixture();
|
||||
let removing: Promise<{ value?: unknown; error?: unknown }> | undefined;
|
||||
try {
|
||||
await db.transaction(async tx => {
|
||||
await tx.execute(sql`set local lock_timeout = '1s'`);
|
||||
await tx.select().from(companies).where(eq(companies.id, f.company.id)).for("no key update");
|
||||
await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)).for("update");
|
||||
removing = services.value.agentService(db).remove(f.agent.id).then(value => ({ value }), error => ({ error }));
|
||||
await vi.waitFor(async () => {
|
||||
const waiting = await db.execute(sql`select query from pg_stat_activity where datname = current_database()
|
||||
and wait_event_type = 'Lock' and (query like '%companies%for no key update%' or query like 'delete from "heartbeat_runs"%')`);
|
||||
expect(waiting.length).toBeGreaterThan(0);
|
||||
}, { timeout: 3000 });
|
||||
await createCostEventInTransaction(tx as unknown as ReturnType<typeof createDb>, f.company.id,
|
||||
{ ...f.receipt, heartbeatRunId: f.run.id }, []);
|
||||
});
|
||||
// Existing charge FKs prohibit hard deletion. Preserve the settled charge
|
||||
// and the atomic deletion failure, rather than deadlocking its writer.
|
||||
expect(await removing!).toMatchObject({ error: { cause: { code: "23503" } } });
|
||||
expect((await costService(db).summary(f.company.id)).spendCents).toBe(100);
|
||||
expect(await db.select().from(agents).where(eq(agents.id, f.agent.id))).toHaveLength(1);
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id))).toHaveLength(1);
|
||||
} finally { await removing; }
|
||||
});
|
||||
|
||||
it("preserves an in-flight native result when company deletion reaches its existing foreign-key restriction", async () => {
|
||||
const f = await fixture();
|
||||
await costService(db).createEvent(f.company.id, { ...f.receipt, heartbeatRunId: f.run.id });
|
||||
const contractHash = randomUUID();
|
||||
const [contract] = await db.insert(completionContracts).values({ companyId: f.company.id, issueId: f.issue.id,
|
||||
revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test",
|
||||
risk: "standard", completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal",
|
||||
contractJson: {}, canonicalSha256: contractHash, createdByActorType: "system", createdByActorId: "test" }).returning();
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: f.issue.id,
|
||||
completionContractId: contract.id, completionContractSha256: contractHash }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
let removing: Promise<{ value?: unknown; error?: unknown }> | undefined;
|
||||
try {
|
||||
await db.transaction(async tx => {
|
||||
await tx.execute(sql`set local lock_timeout = '1s'`);
|
||||
await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)).for("update");
|
||||
removing = services.value.companyService(db).remove(f.company.id).then(value => ({ value }), error => ({ error }));
|
||||
await vi.waitFor(async () => {
|
||||
const waiting = await db.execute(sql`select pid from pg_stat_activity where datname = current_database()
|
||||
and wait_event_type = 'Lock' and query like 'delete from "heartbeat_runs"%'`);
|
||||
expect(waiting.length).toBeGreaterThan(0);
|
||||
}, { timeout: 3000 });
|
||||
const store = new NativeRunCoordinatorStore(tx as unknown as ReturnType<typeof createDb>, {
|
||||
companyId: f.company.id, issueId: f.issue.id, runId: f.run.id, agentId: f.agent.id,
|
||||
normalizedSessionId: randomUUID(), runnerSourceInstanceId: randomUUID(),
|
||||
completionContractId: contract.id, completionContractSha256: contractHash,
|
||||
completionContractRevision: CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim.contractRevision,
|
||||
completionContractCriterionIds: CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim.criteria.map(c => c.criterionId),
|
||||
});
|
||||
expect(await store.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL }))
|
||||
.toMatchObject({ disposition: "committed" });
|
||||
});
|
||||
// Native artifacts already prevent hard deletion via restrictive FKs.
|
||||
// Preserve that atomic failure; the accounting lock must not introduce a
|
||||
// deadlock that aborts the native result instead (Postgres 40P01).
|
||||
expect(await removing!).toMatchObject({ error: { cause: { code: "23503" } } });
|
||||
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.companyId, f.company.id))).toHaveLength(1);
|
||||
expect(await db.select().from(companies).where(eq(companies.id, f.company.id))).toHaveLength(1);
|
||||
expect((await costService(db).summary(f.company.id)).spendCents).toBe(100);
|
||||
} finally { await removing; }
|
||||
});
|
||||
|
||||
it("counts cache reads once across mixed historical and receipt-backed reports without changing charges", async () => {
|
||||
const f = await fixture();
|
||||
const costs = costService(db);
|
||||
const scope = { agentId: f.agent.id, issueId: f.issue.id, projectId: f.project.id, heartbeatRunId: f.run.id,
|
||||
provider: "openai", biller: "openai", model: "gpt-test", occurredAt: new Date() };
|
||||
// These two layouts both describe 100 input (80 cached) and 10 output.
|
||||
for (const billingType of ["metered_api", "subscription_included"]) {
|
||||
await db.insert(costEvents).values({ companyId: f.company.id, ...scope, billingType,
|
||||
inputTokens: 100, cachedInputTokens: 80, outputTokens: 10, costCents: 7 });
|
||||
await costs.createEvent(f.company.id, { ...scope, billingType,
|
||||
inputTokens: 20, cachedInputTokens: 80, outputTokens: 10, costCents: 11 });
|
||||
}
|
||||
const [provider, models] = await Promise.all([costs.byProvider(f.company.id), costs.byAgentModel(f.company.id)]);
|
||||
for (const rows of [provider, models]) {
|
||||
expect(rows).toHaveLength(2);
|
||||
for (const row of rows) {
|
||||
expect(row).toMatchObject({ inputTokens: 40, cachedInputTokens: 160, outputTokens: 20, costCents: 18 });
|
||||
expect(row.inputTokens + row.cachedInputTokens + row.outputTokens).toBe(220);
|
||||
}
|
||||
}
|
||||
const [agent] = await costs.byAgent(f.company.id);
|
||||
const [biller] = await costs.byBiller(f.company.id);
|
||||
const [project] = await costs.byProject(f.company.id);
|
||||
const issue = await costs.issueTreeSummary(f.company.id, f.issue.id);
|
||||
const windows = await costs.windowSpend(f.company.id);
|
||||
expect(windows).toHaveLength(3);
|
||||
for (const row of [agent, biller, project, issue, ...windows]) {
|
||||
expect(row).toMatchObject({ inputTokens: 80, cachedInputTokens: 320, outputTokens: 40, costCents: 36 });
|
||||
expect(row.inputTokens + row.cachedInputTokens + row.outputTokens).toBe(440);
|
||||
}
|
||||
for (const row of [agent, biller, provider.find(row => row.billingType === "subscription_included")!]) {
|
||||
expect(row).toMatchObject({ subscriptionInputTokens: 40, subscriptionCachedInputTokens: 160, subscriptionOutputTokens: 20 });
|
||||
}
|
||||
// Reading reports never rewrites historical evidence or retroactively bills.
|
||||
const stored = await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id));
|
||||
expect(stored.filter(row => row.receiptHash === null).map(row => row.inputTokens)).toEqual([100, 100]);
|
||||
expect((await costs.summary(f.company.id)).spendCents).toBe(36);
|
||||
// Legacy Anthropic input was already exclusive; retain its cache semantics.
|
||||
await db.insert(costEvents).values({ companyId: f.company.id, ...scope, provider: "anthropic", billingType: "metered_api",
|
||||
inputTokens: 100, cachedInputTokens: 80, outputTokens: 10, costCents: 0 });
|
||||
expect((await costs.byProvider(f.company.id)).find(row => row.provider === "anthropic")).toMatchObject({ inputTokens: 100, cachedInputTokens: 80, outputTokens: 10 });
|
||||
});
|
||||
|
||||
it.each([
|
||||
["openai/gpt-test", 100], ["openai/vendor/model", 100],
|
||||
["gpt-test", 20], ["different/model", 20], ["openai/", 20],
|
||||
] as const)("preserves historical input semantics for %s", async (model, inputTokens) => {
|
||||
const f = await fixture(), costs = costService(db);
|
||||
await db.insert(costEvents).values({ companyId: f.company.id, agentId: f.agent.id,
|
||||
issueId: f.issue.id, projectId: f.project.id, heartbeatRunId: f.run.id,
|
||||
provider: "openai", biller: "openai", model, occurredAt: new Date(),
|
||||
inputTokens: 100, cachedInputTokens: 80, outputTokens: 10, costCents: 7 });
|
||||
const rows = [
|
||||
...await costs.byAgent(f.company.id), ...await costs.byProvider(f.company.id),
|
||||
...await costs.byAgentModel(f.company.id), ...await costs.byBiller(f.company.id),
|
||||
...await costs.byProject(f.company.id), ...await costs.windowSpend(f.company.id),
|
||||
await costs.issueTreeSummary(f.company.id, f.issue.id),
|
||||
...(await costs.byUser(f.company.id)).rows,
|
||||
];
|
||||
for (const row of rows) expect(row).toMatchObject({ inputTokens, cachedInputTokens: 80, outputTokens: 10, costCents: 7 });
|
||||
expect((await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id)))[0].inputTokens).toBe(100);
|
||||
});
|
||||
|
||||
it.each((["company", "agent", "project"] as const).flatMap(scopeType => [0, 500].map(previousAmount => ({ scopeType, previousAmount }))))("edits $scopeType limits without confusing saved zero and disabled positive amounts ($previousAmount)", async ({ scopeType, previousAmount }) => {
|
||||
const f = await fixture(), cancel = vi.fn(async () => {});
|
||||
const budgets = budgetService(db, { cancelWorkForScope: cancel });
|
||||
const scopeId = scopeType === "company" ? f.company.id : scopeType === "agent" ? f.agent.id : f.project.id;
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: previousAmount, isActive: false }, "board");
|
||||
await costService(db).createEvent(f.company.id, { ...f.receipt, projectId: f.project.id, costCents: 100 });
|
||||
expect((await budgets.overview(f.company.id)).policies[0]).toMatchObject({ amount: 0, isActive: false });
|
||||
const updated = await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: 50 }, "board");
|
||||
expect(updated).toMatchObject({ isActive: previousAmount === 0, paused: previousAmount === 0 });
|
||||
expect(cancel).toHaveBeenCalledTimes(previousAmount === 0 ? 1 : 0);
|
||||
expect((await budgets.overview(f.company.id)).activeIncidents).toHaveLength(previousAmount === 0 ? 1 : 0);
|
||||
});
|
||||
|
||||
it.each(["issueId", "projectId", "goalId", "heartbeatRunId"] as const)("rejects a foreign-company %s without writing anything", async (field) => {
|
||||
const a = await fixture(); const b = await fixture();
|
||||
const foreignIds = { issueId: b.issue.id, projectId: b.project.id, goalId: b.goal.id, heartbeatRunId: b.run.id };
|
||||
await expect(costService(db).createEvent(a.company.id, { ...a.receipt, [field]: foreignIds[field] })).rejects.toThrow(/company/);
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, a.company.id))).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("allows admission inside a caller transaction holding a company foreign-key lock", async () => {
|
||||
const f = await fixture();
|
||||
// The recovery service dispatches through another connection while its
|
||||
// transaction still owns FK KEY SHARE locks on the company. Accounting
|
||||
// must serialize without requesting the conflicting FOR UPDATE mode.
|
||||
await db.transaction(async (tx) => {
|
||||
await tx.insert(issues).values({ companyId: f.company.id, title: "Concurrent recovery" });
|
||||
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
|
||||
});
|
||||
}, 5000);
|
||||
|
||||
it.each([false, true])("takes the attributed issue lock before the run during native recovery (attribution changes: %s)", async (changeAttribution) => {
|
||||
const f = await fixture();
|
||||
const [nextIssue] = await db.insert(issues).values({ companyId: f.company.id, title: "New attribution" }).returning();
|
||||
const usage = { provider: "openai", billingType: "metered_api", costUsd: 0.5, inputTokens: 10, outputTokens: 2,
|
||||
accountingReceiptReady: true, ledgerScope: { issueId: f.issue.id } };
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: f.issue.id, status: "failed", finishedAt: new Date(),
|
||||
costAccountingPending: true, usageJson: usage }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
let accounting: Promise<{ value?: boolean; error?: unknown }> | undefined;
|
||||
try {
|
||||
await db.transaction(async tx => {
|
||||
// recordRetryableFailure holds the issue while updating its run.
|
||||
await tx.select().from(issues).where(eq(issues.id, f.issue.id)).for("update");
|
||||
accounting = accountRunCost(db, f.run.id).then(value => ({ value }), error => ({ error }));
|
||||
await vi.waitFor(async () => {
|
||||
const waiting = await db.execute(sql`select pid from pg_stat_activity where datname = current_database()
|
||||
and wait_event_type = 'Lock' and query like '%issues%for key share%'`);
|
||||
expect(waiting.length).toBeGreaterThan(0);
|
||||
}, { timeout: 3000 });
|
||||
await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)).for("update", { noWait: true });
|
||||
await tx.update(heartbeatRuns).set({ resultJson: { prpRunTerminalState: "failed" },
|
||||
...(changeAttribution ? { usageJson: { ...usage, ledgerScope: { issueId: nextIssue.id } } } : {}),
|
||||
}).where(eq(heartbeatRuns.id, f.run.id));
|
||||
});
|
||||
if (changeAttribution) {
|
||||
expect((await accounting!).error).toMatchObject({ status: 409, message: "Run cost attribution changed. Retry accounting." });
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toHaveLength(0);
|
||||
expect(await accountRunCost(db, f.run.id)).toBe(true);
|
||||
} else expect(await accounting!).toEqual({ value: true });
|
||||
const charges = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id));
|
||||
expect(charges).toHaveLength(1);
|
||||
expect(charges[0]).toMatchObject({ issueId: changeAttribution ? nextIssue.id : f.issue.id, costCents: 50 });
|
||||
} finally { await accounting; }
|
||||
});
|
||||
|
||||
it("takes the company lock before retry issue/run locks during an attributed cost write", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(issues).set({ status: "in_progress", assigneeAgentId: f.agent.id }).where(eq(issues.id, f.issue.id));
|
||||
await db.update(heartbeatRuns).set({ status: "scheduled_retry", scheduledRetryAt: new Date(0), scheduledRetryReason: "workspace_busy",
|
||||
contextSnapshot: { issueId: f.issue.id, projectId: f.project.id },
|
||||
}).where(eq(heartbeatRuns.id, f.run.id));
|
||||
let promotion: ReturnType<ReturnType<typeof createRunDispatch>["promoteScheduledRetry"]> | undefined;
|
||||
await db.transaction(async (tx) => {
|
||||
await tx.select().from(companies).where(eq(companies.id, f.company.id)).for("no key update");
|
||||
promotion = createRunDispatch(db).promoteScheduledRetry({ companyId: f.company.id, runId: f.run.id });
|
||||
// Observe the actual blocked company-lock query before checking the issue;
|
||||
// no timing guess or timeout is used as proof of lock ordering.
|
||||
await vi.waitFor(async () => {
|
||||
const result = await db.execute(sql`select pid from pg_stat_activity where datname = current_database()
|
||||
and pid <> pg_backend_pid() and wait_event_type = 'Lock' and query like '%companies%for no key update%'`);
|
||||
expect(result.length).toBeGreaterThan(0);
|
||||
}, { timeout: 3000 });
|
||||
await tx.execute(sql`select id from issues where id = ${f.issue.id} for key share nowait`);
|
||||
await createCostEventInTransaction(tx as unknown as ReturnType<typeof createDb>, f.company.id,
|
||||
{ ...f.receipt, issueId: f.issue.id, projectId: f.project.id }, []);
|
||||
});
|
||||
expect((await promotion)?.outcome).toBe("promoted");
|
||||
expect((await costService(db).summary(f.company.id)).spendCents).toBe(100);
|
||||
}, 10000);
|
||||
|
||||
it("locks company before activating an approved hire while a receipt writer is active", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(agents).set({ status: "pending_approval" }).where(eq(agents.id, f.agent.id));
|
||||
const [approval] = await db.insert(approvals).values({ companyId: f.company.id, type: "hire_agent", status: "pending",
|
||||
payload: { agentId: f.agent.id, budgetMonthlyCents: 1000 } }).returning();
|
||||
let approving: Promise<unknown> | undefined;
|
||||
await db.transaction(async tx => {
|
||||
await tx.select().from(companies).where(eq(companies.id, f.company.id)).for("no key update");
|
||||
approving = services.value.approvalService(db).approve(approval.id, "board");
|
||||
await vi.waitFor(async () => {
|
||||
const waiting = await db.execute(sql`select pid from pg_stat_activity where datname = current_database()
|
||||
and pid <> pg_backend_pid() and wait_event_type = 'Lock' and query like '%companies%for no key update%'`);
|
||||
expect(waiting.length).toBeGreaterThan(0);
|
||||
}, { timeout: 3000 });
|
||||
await tx.execute(sql`select id from agents where id = ${f.agent.id} for no key update nowait`);
|
||||
await createCostEventInTransaction(tx as unknown as ReturnType<typeof createDb>, f.company.id, f.receipt, []);
|
||||
});
|
||||
await expect(approving).resolves.toMatchObject({ applied: true });
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("idle");
|
||||
expect((await costService(db).summary(f.company.id)).spendCents).toBe(100);
|
||||
}, 10000);
|
||||
|
||||
it("deduplicates concurrent retries and rejects changed receipt content", async () => {
|
||||
const f = await fixture();
|
||||
const receipt = { ...f.receipt, idempotencyKey: "provider-receipt-1" };
|
||||
const events = await Promise.all(Array.from({ length: 8 }, () => costService(db).createEvent(f.company.id, receipt)));
|
||||
expect(new Set(events.map((event) => event.id)).size).toBe(1);
|
||||
expect((await costService(db).summary(f.company.id)).spendCents).toBe(100);
|
||||
await expect(costService(db).createEvent(f.company.id, { ...receipt, costCents: 200 })).rejects.toThrow(/different|conflict/i);
|
||||
// Keys are scoped to the reporting company.
|
||||
const other = await fixture();
|
||||
await expect(costService(db).createEvent(other.company.id, { ...other.receipt, idempotencyKey: receipt.idempotencyKey })).resolves.toBeTruthy();
|
||||
});
|
||||
|
||||
it("retains sub-cent spend and enforces the sum, not rounded individual receipts", async () => {
|
||||
const f = await fixture(); const budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 1 }, "board");
|
||||
for (let i = 0; i < 3; i++) await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 0.4 });
|
||||
expect((await costService(db).summary(f.company.id)).spendCents).toBe(1.2);
|
||||
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toMatchObject({ scopeType: "agent" });
|
||||
});
|
||||
|
||||
it("conserves project totals when a run touches multiple projects and includes unallocated spend", async () => {
|
||||
const f = await fixture();
|
||||
const [otherProject] = await db.insert(projects).values({ companyId: f.company.id, name: "Other" }).returning();
|
||||
const [otherIssue] = await db.insert(issues).values({ companyId: f.company.id, projectId: otherProject.id, title: "Other" }).returning();
|
||||
await db.insert(activityLog).values([f.issue, otherIssue].map((issue) => ({ companyId: f.company.id, actorType: "agent", actorId: f.agent.id, action: "issue.updated", entityType: "issue", entityId: issue.id, runId: f.run.id })));
|
||||
await costService(db).createEvent(f.company.id, { ...f.receipt, heartbeatRunId: f.run.id, projectId: f.project.id });
|
||||
await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 30 });
|
||||
const rows = await costService(db).byProject(f.company.id);
|
||||
expect(rows.reduce((sum, row) => sum + row.costCents, 0)).toBe(130);
|
||||
expect(rows.find((row) => row.projectId === f.project.id)?.costCents).toBe(100);
|
||||
expect(rows.find((row) => row.projectId === null)?.costCents).toBe(30);
|
||||
});
|
||||
|
||||
it("does not enforce inactive policies, and checks both monthly and lifetime limits", async () => {
|
||||
const f = await fixture(); const budgets = budgetService(db);
|
||||
await costService(db).createEvent(f.company.id, f.receipt);
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 1, isActive: false }, "board");
|
||||
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 1000, windowKind: "calendar_month_utc" }, "board");
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 50, windowKind: "lifetime" }, "board");
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 2000, windowKind: "calendar_month_utc" }, "board");
|
||||
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toMatchObject({ scopeType: "agent" });
|
||||
});
|
||||
|
||||
it("creates one approval under concurrent evaluation and opens a new incident after a raised budget is reached", async () => {
|
||||
const f = await fixture(); const budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100 }, "board");
|
||||
const event = await costService(db).createEvent(f.company.id, f.receipt);
|
||||
await Promise.all(Array.from({ length: 8 }, () => budgets.evaluateCostEvent(event)));
|
||||
expect(await db.select().from(approvals).where(eq(approvals.companyId, f.company.id))).toHaveLength(1);
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 150 }, "board");
|
||||
await costService(db).createEvent(f.company.id, f.receipt);
|
||||
expect(await db.select().from(budgetIncidents).where(and(eq(budgetIncidents.companyId, f.company.id), eq(budgetIncidents.thresholdType, "hard"), eq(budgetIncidents.status, "open")))).toHaveLength(1);
|
||||
expect(await db.select().from(approvals).where(eq(approvals.companyId, f.company.id))).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("commits accounting despite cancellation delivery failure and retries delivery safely", async () => {
|
||||
const f = await fixture();
|
||||
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100 }, "board");
|
||||
const cancel = vi.fn().mockRejectedValueOnce(new Error("injected cancellation transport failure")).mockResolvedValue(undefined);
|
||||
const costs = costService(db, { cancelWorkForScope: cancel });
|
||||
const receipt = { ...f.receipt, idempotencyKey: "retry-after-cancel" };
|
||||
await expect(costs.createEvent(f.company.id, receipt)).resolves.toBeTruthy();
|
||||
await costs.createEvent(f.company.id, receipt);
|
||||
expect(cancel).toHaveBeenCalledTimes(2);
|
||||
expect((await costs.summary(f.company.id)).spendCents).toBe(100);
|
||||
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toMatchObject({ scopeType: "agent" });
|
||||
});
|
||||
|
||||
it("recovers live scopes despite deleted agent and project policies and an isolated transaction failure", async () => {
|
||||
const f = await fixture(); const other = await fixture(); const budgets = budgetService(db);
|
||||
const [removedAgent] = await db.insert(agents).values({ companyId: f.company.id, name: "Removed", adapterType: "process" }).returning();
|
||||
const [removedProject] = await db.insert(projects).values({ companyId: f.company.id, name: "Removed" }).returning();
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: removedAgent.id, amount: 100 }, "board");
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "project", scopeId: removedProject.id, amount: 100 }, "board");
|
||||
await db.delete(agents).where(eq(agents.id, removedAgent.id));
|
||||
await db.delete(projects).where(eq(projects.id, removedProject.id));
|
||||
for (const live of [f, other]) {
|
||||
await budgets.upsertPolicy(live.company.id, { scopeType: "agent", scopeId: live.agent.id, amount: 100 }, "board");
|
||||
await db.update(agents).set({ status: "paused", pauseReason: "budget" }).where(eq(agents.id, live.agent.id));
|
||||
}
|
||||
// Restrict the scanned scope list so the injected first transaction fails
|
||||
// on the stale scope, ahead of both live policies in this real database.
|
||||
const query = vi.spyOn(db, "selectDistinct").mockReturnValueOnce({ from: () => ({ orderBy: async () => [
|
||||
{ companyId: f.company.id, scopeType: "project", scopeId: removedProject.id },
|
||||
{ companyId: f.company.id, scopeType: "agent", scopeId: removedAgent.id },
|
||||
...[f, other].map(live => ({ companyId: live.company.id, scopeType: "agent", scopeId: live.agent.id })),
|
||||
] }) } as never);
|
||||
const transaction = vi.spyOn(db, "transaction").mockRejectedValueOnce(new Error("Injected scope recovery failure"));
|
||||
try { await budgets.reconcilePolicies(); } finally { query.mockRestore(); transaction.mockRestore(); }
|
||||
for (const live of [f, other]) {
|
||||
const [agent] = await db.select().from(agents).where(eq(agents.id, live.agent.id));
|
||||
expect(agent).toMatchObject({ status: "idle", pauseReason: null });
|
||||
}
|
||||
// A subsequent sweep uses actual policy discovery and skips both deleted scopes.
|
||||
await expect(budgets.reconcilePolicies()).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("recovers terminal run accounting exactly once across simultaneous live and recovery workers", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(heartbeatRuns).set({ status: "timed_out", finishedAt: new Date(), costAccountingPending: true,
|
||||
usageJson: { inputTokens: 20, cachedInputTokens: 100, outputTokens: 10, costUsd: 0.004, provider: "test", model: "test", billingType: "metered_api", ledgerScope: { issueId: f.issue.id, projectId: f.project.id } },
|
||||
}).where(eq(heartbeatRuns.id, f.run.id));
|
||||
await Promise.all([accountRunCost(db, f.run.id), accountRunCost(db, f.run.id), reconcileRunCosts(db)]);
|
||||
const [totals] = await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id));
|
||||
expect(totals).toMatchObject({ totalInputTokens: 20, totalCachedInputTokens: 100, totalOutputTokens: 10, totalCostCents: 0.4 });
|
||||
const events = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id));
|
||||
expect(events).toHaveLength(1);
|
||||
expect(events[0]).toMatchObject({ costCents: 0.4, projectId: f.project.id });
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id));
|
||||
expect(run.costAccountingPending).toBe(false);
|
||||
expect(run.costAccountedAt).toBeInstanceOf(Date);
|
||||
// Late terminal metadata may re-arm the pending flag. The durable
|
||||
// acknowledgement, not the delivery flag, owns exactly-once totals.
|
||||
await db.update(heartbeatRuns).set({ costAccountingPending: true }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
await accountRunCost(db, f.run.id);
|
||||
expect((await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id)))[0].totalCostCents).toBe(0.4);
|
||||
});
|
||||
|
||||
it("rolls back the ledger and counters on accounting failure, retaining a retryable run", async () => {
|
||||
const f = await fixture(); const other = await fixture();
|
||||
await db.update(heartbeatRuns).set({ status: "failed", costAccountingPending: true, usageJson: {
|
||||
inputTokens: 10, costUsd: 1, provider: "test", model: "test", ledgerScope: { projectId: other.project.id },
|
||||
} }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
await expect(accountRunCost(db, f.run.id)).rejects.toThrow(/company/);
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(0);
|
||||
expect(await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id))).toHaveLength(0);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountingPending).toBe(true);
|
||||
});
|
||||
|
||||
it("rolls back a receipt, counters, and budget incidents if a later runtime write fails", async () => {
|
||||
const f = await fixture();
|
||||
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 50 }, "board");
|
||||
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date(), costAccountingPending: true, usageJson: {
|
||||
inputTokens: 10, costUsd: 1, provider: "test", billingType: "metered_api", model: "test",
|
||||
} }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
await db.execute(sql`create function reject_runtime_accounting() returns trigger language plpgsql as $$ begin raise exception 'injected runtime write failure'; end $$`);
|
||||
await db.execute(sql`create trigger reject_runtime_accounting before insert on agent_runtime_state for each row execute function reject_runtime_accounting()`);
|
||||
try {
|
||||
await expect(accountRunCost(db, f.run.id)).rejects.toThrow();
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(0);
|
||||
expect(await db.select().from(budgetIncidents).where(eq(budgetIncidents.companyId, f.company.id))).toHaveLength(0);
|
||||
expect(await db.select().from(approvals).where(eq(approvals.companyId, f.company.id))).toHaveLength(0);
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0]).toMatchObject({ spentMonthlyCents: 0, status: "active" });
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]).toMatchObject({ costAccountingPending: true, costAccountedAt: null });
|
||||
} finally {
|
||||
await db.execute(sql`drop trigger reject_runtime_accounting on agent_runtime_state`);
|
||||
await db.execute(sql`drop function reject_runtime_accounting()`);
|
||||
}
|
||||
await accountRunCost(db, f.run.id);
|
||||
expect((await costService(db).summary(f.company.id)).spendCents).toBe(100);
|
||||
});
|
||||
|
||||
it("releases an expired monthly budget pause while preserving manual pauses", async () => {
|
||||
const f = await fixture(); const budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 50 }, "board");
|
||||
const event = await costService(db).createEvent(f.company.id, f.receipt);
|
||||
const now = new Date();
|
||||
await db.update(costEvents).set({ occurredAt: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 0)) }).where(eq(costEvents.id, event.id));
|
||||
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0]).toMatchObject({ status: "idle", pauseReason: null });
|
||||
await db.update(agents).set({ status: "paused", pauseReason: "manual" }).where(eq(agents.id, f.agent.id));
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 200 }, "board");
|
||||
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0]).toMatchObject({ status: "paused", pauseReason: "manual" });
|
||||
});
|
||||
|
||||
it("uses live observed spend for incident resolution and synchronizes company budgets", async () => {
|
||||
const f = await fixture(); const budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 50 }, "board");
|
||||
await costService(db).createEvent(f.company.id, f.receipt);
|
||||
const [incident] = await db.select().from(budgetIncidents).where(and(eq(budgetIncidents.companyId, f.company.id), eq(budgetIncidents.thresholdType, "hard")));
|
||||
await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 25 });
|
||||
await expect(budgets.resolveIncident(f.company.id, incident.id, { action: "raise_budget_and_resume", amount: 125 }, "board")).rejects.toThrow("New budget must exceed current observed spend");
|
||||
await budgets.resolveIncident(f.company.id, incident.id, { action: "raise_budget_and_resume", amount: 175 }, "board");
|
||||
expect((await db.select().from(companies).where(eq(companies.id, f.company.id)))[0]).toMatchObject({ budgetMonthlyCents: 175, status: "active", pauseReason: null });
|
||||
});
|
||||
|
||||
it("exposes missing prices and blocks strict budgets until explicitly allowed", async () => {
|
||||
const f = await fixture(); const budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100 }, "board");
|
||||
await costService(db).createEvent(f.company.id, { ...f.receipt, inputTokens: 500, costCents: 0, costStatus: "unpriced" });
|
||||
expect(await costService(db).summary(f.company.id)).toMatchObject({ spendCents: 0, unpricedEventCount: 1, pricingComplete: false });
|
||||
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toMatchObject({ reason: "Agent cannot start work because recorded usage has no reliable price." });
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100, unpricedUsagePolicy: "allow" }, "board");
|
||||
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
|
||||
expect((await budgets.overview(f.company.id)).policies[0]).toMatchObject({ unpricedEventCount: 1, unpricedUsagePolicy: "allow" });
|
||||
});
|
||||
|
||||
it("keeps subscription-included usage outside monetary pricing gaps", async () => {
|
||||
const f = await fixture(); const budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100 }, "board");
|
||||
await costService(db).createEvent(f.company.id, { ...f.receipt, inputTokens: 500, costCents: 0, costStatus: "unpriced", billingType: "subscription_included" });
|
||||
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
|
||||
expect(await costService(db).summary(f.company.id)).toMatchObject({ pricingComplete: true });
|
||||
});
|
||||
|
||||
it("defaults reports to the UTC month and excludes future events from rolling spend", async () => {
|
||||
const f = await fixture(); const costs = costService(db); const now = new Date();
|
||||
await costs.createEvent(f.company.id, { ...f.receipt, occurredAt: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - 1, 1)) });
|
||||
await costs.createEvent(f.company.id, { ...f.receipt, occurredAt: new Date(now.getTime() + 86_400_000) });
|
||||
expect((await costs.summary(f.company.id)).spendCents).toBe(0);
|
||||
expect((await costs.summary(f.company.id, { allTime: true })).spendCents).toBe(200);
|
||||
expect(await costs.windowSpend(f.company.id)).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps finance currencies separate and deduplicates invoice-line retries", async () => {
|
||||
const f = await fixture(); const finance = financeService(db);
|
||||
const usd = { biller: "vendor", eventKind: "platform_fee", amountCents: 100, currency: "usd", occurredAt: new Date(), idempotencyKey: "invoice:line:1", metadataJson: { a: 1, b: 2 } };
|
||||
await Promise.all(Array.from({ length: 6 }, () => finance.createEvent(f.company.id, usd)));
|
||||
await finance.createEvent(f.company.id, { ...usd, metadataJson: { b: 2, a: 1 } });
|
||||
await finance.createEvent(f.company.id, { ...usd, currency: "EUR", idempotencyKey: "invoice:line:2" });
|
||||
const summary = await finance.summary(f.company.id);
|
||||
expect(summary).toMatchObject({ currency: "USD", debitCents: 100, netCents: 100 });
|
||||
expect(summary.currencies).toEqual(expect.arrayContaining([expect.objectContaining({ currency: "USD", netCents: 100 }), expect.objectContaining({ currency: "EUR", netCents: 100 })]));
|
||||
expect(await finance.byBiller(f.company.id)).toHaveLength(2);
|
||||
expect(await finance.byKind(f.company.id)).toHaveLength(2);
|
||||
expect(await db.select().from(financeEvents).where(eq(financeEvents.companyId, f.company.id))).toHaveLength(2);
|
||||
await expect(finance.createEvent(f.company.id, { ...usd, amountCents: 200 })).rejects.toThrow(/different/);
|
||||
});
|
||||
|
||||
it("synchronizes budgets through generic agent and company updates", async () => {
|
||||
const f = await fixture();
|
||||
const { agentService, companyService } = services.value;
|
||||
await agentService(db).update(f.agent.id, { budgetMonthlyCents: 50 });
|
||||
await companyService(db).update(f.company.id, { budgetMonthlyCents: 75 });
|
||||
await costService(db).createEvent(f.company.id, f.receipt);
|
||||
const policies = await budgetService(db).listPolicies(f.company.id);
|
||||
expect(policies).toEqual(expect.arrayContaining([expect.objectContaining({ scopeType: "agent", amount: 50 }), expect.objectContaining({ scopeType: "company", amount: 75 })]));
|
||||
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toMatchObject({ scopeType: "company" });
|
||||
});
|
||||
|
||||
it("delivers cancellation after generic budget updates commit", async () => {
|
||||
const f = await fixture();
|
||||
await costService(db).createEvent(f.company.id, f.receipt);
|
||||
const cancelWorkForScope = vi.fn(async (scope) => {
|
||||
const policies = await budgetService(db).listPolicies(f.company.id);
|
||||
expect(policies.some(policy => policy.scopeType === scope.scopeType && policy.amount < 100)).toBe(true);
|
||||
});
|
||||
await services.value.agentService(db, { cancelWorkForScope }).update(f.agent.id, { budgetMonthlyCents: 50 });
|
||||
await services.value.companyService(db, { cancelWorkForScope }).update(f.company.id, { budgetMonthlyCents: 75 });
|
||||
expect(cancelWorkForScope).toHaveBeenCalledWith(expect.objectContaining({ scopeType: "agent", scopeId: f.agent.id }));
|
||||
expect(cancelWorkForScope).toHaveBeenCalledWith(expect.objectContaining({ scopeType: "company", scopeId: f.company.id }));
|
||||
const rows = await db.select().from(budgetPolicies).where(eq(budgetPolicies.companyId, f.company.id));
|
||||
expect(rows.every(row => row.enforcementVersion === row.enforcementDeliveredVersion)).toBe(true);
|
||||
});
|
||||
|
||||
it("merges partial policy edits with current settings under the accounting lock", async () => {
|
||||
const f = await fixture(); const budgets = budgetService(db);
|
||||
const identity = { scopeType: "agent" as const, scopeId: f.agent.id };
|
||||
await budgets.upsertPolicy(f.company.id, { ...identity, amount: 100, hardStopEnabled: true }, "board");
|
||||
await budgets.upsertPolicy(f.company.id, { ...identity, amount: 200, hardStopEnabled: false, warnPercent: 60, notifyEnabled: false }, "other-operator");
|
||||
const saved = await budgets.upsertPolicy(f.company.id, upsertBudgetPolicySchema.parse({ ...identity, reservationCents: "10" }), "board");
|
||||
expect(saved).toMatchObject({ amount: 200, hardStopEnabled: false, warnPercent: 60, notifyEnabled: false, reservationCents: "10.0000000" });
|
||||
await budgets.upsertPolicy(f.company.id, { ...identity, isActive: false }, "other-operator");
|
||||
const disabled = await budgets.upsertPolicy(f.company.id, upsertBudgetPolicySchema.parse({ ...identity, amount: 300 }), "board");
|
||||
expect(await db.select({ amount: budgetPolicies.amount, isActive: budgetPolicies.isActive }).from(budgetPolicies).where(eq(budgetPolicies.id, disabled.policyId))).toEqual([{ amount: 300, isActive: false }]);
|
||||
expect(disabled).toMatchObject({ amount: 0, isActive: false, hardStopEnabled: false, warnPercent: 60, notifyEnabled: false, reservationCents: "10.0000000" });
|
||||
await expect(budgets.upsertPolicy(f.company.id, { scopeType: "project", scopeId: f.project.id, reservationCents: "1" }, "board")).rejects.toThrow("Amount is required");
|
||||
});
|
||||
|
||||
it("conserves both spend and tokens across complete per-model receipts", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date(), costAccountingPending: true, usageJson: {
|
||||
provider: "anthropic", billingType: "metered_api", model: "mixed", costUsd: 0.007,
|
||||
inputTokens: 30, outputTokens: 4, cachedInputTokens: 100,
|
||||
usageByModel: [
|
||||
{ model: "large", costUsd: 0.005, usage: { inputTokens: 20, outputTokens: 3, cachedInputTokens: 100 } },
|
||||
{ model: "small", costUsd: 0.002, usage: { inputTokens: 10, outputTokens: 1 } },
|
||||
],
|
||||
} }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
await accountRunCost(db, f.run.id);
|
||||
const rows = await costService(db).byProvider(f.company.id);
|
||||
expect(rows).toEqual(expect.arrayContaining([expect.objectContaining({ model: "large", costCents: 0.5, inputTokens: 20 }), expect.objectContaining({ model: "small", costCents: 0.2, inputTokens: 10 })]));
|
||||
expect((await costService(db).summary(f.company.id)).spendCents).toBe(0.7);
|
||||
const [totals] = await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id));
|
||||
expect(totals).toMatchObject({ totalInputTokens: 30, totalCachedInputTokens: 100, totalOutputTokens: 4, totalCostCents: 0.7 });
|
||||
});
|
||||
it("blocks admission while a terminal run awaits accounting, even under an allow-unpriced policy", async () => {
|
||||
const f = await fixture(); const budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 1000, unpricedUsagePolicy: "allow" }, "board");
|
||||
await db.update(heartbeatRuns).set({ status: "failed", costAccountingPending: true }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toMatchObject({ reason: expect.stringContaining("await accounting") });
|
||||
expect(await costService(db).summary(f.company.id)).toMatchObject({ pricingComplete: false, pendingRunCount: 1 });
|
||||
await accountRunCost(db, f.run.id);
|
||||
expect(await costService(db).summary(f.company.id)).toMatchObject({ pricingComplete: false, pendingRunCount: 0, unpricedEventCount: 1 });
|
||||
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
|
||||
});
|
||||
|
||||
it("waits for a late provider receipt after cancellation instead of acknowledging a zero charge", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled", costAccountingPending: true, usageJson: { accountingReceiptReady: false } }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
expect(await accountRunCost(db, f.run.id)).toBe(false);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]).toMatchObject({ costAccountingPending: true, costAccountedAt: null });
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toHaveLength(0);
|
||||
await db.update(heartbeatRuns).set({ usageJson: { accountingReceiptReady: true, costUsd: 0.25, inputTokens: 10 } }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
await Promise.all([accountRunCost(db, f.run.id), accountRunCost(db, f.run.id)]);
|
||||
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id)))[0]).toMatchObject({ costCents: 25, inputTokens: 10 });
|
||||
expect((await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id)))[0].totalCostCents).toBe(25);
|
||||
});
|
||||
|
||||
it("acknowledges proven pre-provider failures without inventing a charge", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(heartbeatRuns).set({ status: "failed", costAccountingPending: true,
|
||||
resultJson: { executionRecovery: { providerWorkStarted: false } },
|
||||
}).where(eq(heartbeatRuns.id, f.run.id));
|
||||
await accountRunCost(db, f.run.id);
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(0);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]).toMatchObject({ costAccountingPending: false, costAccountedAt: expect.any(Date) });
|
||||
});
|
||||
|
||||
it("records spend after attribution targets are deleted", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(heartbeatRuns).set({ status: "failed", costAccountingPending: true, usageJson: {
|
||||
costUsd: 0.01, ledgerScope: { projectId: f.project.id, issueId: f.issue.id },
|
||||
} }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
await db.delete(issues).where(eq(issues.id, f.issue.id));
|
||||
await db.delete(projects).where(eq(projects.id, f.project.id));
|
||||
await accountRunCost(db, f.run.id);
|
||||
expect((await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id)))[0]).toMatchObject({ costCents: 1, issueId: null, projectId: null });
|
||||
});
|
||||
|
||||
it("preserves dismissed incidents and warning incidents during admission checks", async () => {
|
||||
const f = await fixture(); const budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100 }, "board");
|
||||
await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 80 });
|
||||
for (let i = 0; i < 3; i++) expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
|
||||
expect((await budgets.overview(f.company.id)).activeIncidents).toHaveLength(1);
|
||||
await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 20 });
|
||||
const [incident] = await db.select().from(budgetIncidents).where(and(eq(budgetIncidents.companyId, f.company.id), eq(budgetIncidents.thresholdType, "hard")));
|
||||
await budgets.resolveIncident(f.company.id, incident.id, { action: "keep_paused" }, "board");
|
||||
for (let i = 0; i < 3; i++) expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeTruthy();
|
||||
expect(await db.select().from(approvals).where(eq(approvals.companyId, f.company.id))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("does not revive terminated agents or archived companies during budget reconciliation", async () => {
|
||||
const f = await fixture(); const budgets = budgetService(db);
|
||||
await db.update(agents).set({ status: "terminated", pauseReason: "budget" }).where(eq(agents.id, f.agent.id));
|
||||
await db.update(companies).set({ status: "archived", pauseReason: "budget" }).where(eq(companies.id, f.company.id));
|
||||
for (const [scopeType, scopeId] of [["agent", f.agent.id], ["company", f.company.id]] as const) {
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: 1000 }, "board");
|
||||
}
|
||||
await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 2000 });
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("terminated");
|
||||
expect((await db.select().from(companies).where(eq(companies.id, f.company.id)))[0].status).toBe("archived");
|
||||
});
|
||||
|
||||
it.each(["agent", "company", "project"] as const)("limits delayed %s budget cancellation to work that preceded the policy check", async (scopeType) => {
|
||||
const f = await fixture();
|
||||
const { heartbeatService } = await import("../services/heartbeat.js");
|
||||
const heartbeat = heartbeatService(db);
|
||||
const scopeId = scopeType === "agent" ? f.agent.id : scopeType === "company" ? f.company.id : f.project.id;
|
||||
const cutoff = new Date();
|
||||
await db.update(heartbeatRuns).set({ status: "queued", createdAt: new Date(cutoff.getTime() - 1000), contextSnapshot: { projectId: f.project.id } }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
const [newRun] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, invocationSource: "on_demand", status: "scheduled_retry",
|
||||
scheduledRetryAt: new Date(cutoff.getTime() + 60000), createdAt: new Date(cutoff.getTime() + 1000), contextSnapshot: { projectId: f.project.id },
|
||||
}).returning();
|
||||
await heartbeat.cancelBudgetScopeWork({ companyId: f.company.id, scopeType, scopeId, createdBefore: cutoff });
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].status).toBe("cancelled");
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, newRun.id)))[0].status).toBe("scheduled_retry");
|
||||
});
|
||||
|
||||
it.each(["agent", "company", "project"] as const)("rechecks delayed %s enforcement after a grant admits an old queued run", async (scopeType) => {
|
||||
const f = await fixture();
|
||||
const { heartbeatService } = await import("../services/heartbeat.js");
|
||||
const heartbeat = heartbeatService(db);
|
||||
const budgets = budgetService(db);
|
||||
const scopeId = scopeType === "agent" ? f.agent.id : scopeType === "company" ? f.company.id : f.project.id;
|
||||
const policyInput = { scopeType, scopeId, amount: 1 };
|
||||
await db.update(heartbeatRuns).set({ status: "queued", contextSnapshot: { projectId: f.project.id } }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.company.id, agentId: f.agent.id, source: "automation", payload: { projectId: f.project.id } }).returning();
|
||||
await budgets.upsertPolicy(f.company.id, policyInput, "board");
|
||||
await costService(db).createEvent(f.company.id, { ...f.receipt, projectId: f.project.id, costCents: 2 });
|
||||
let release!: () => void;
|
||||
const paused = new Promise<void>(resolve => { release = resolve; });
|
||||
let snapshotReady!: (scope: BudgetEnforcementScope) => void;
|
||||
const snapshot = new Promise<BudgetEnforcementScope>(resolve => { snapshotReady = resolve; });
|
||||
const delivery = budgetService(db, { cancelWorkForScope: async (scope) => {
|
||||
snapshotReady(scope); await paused; await heartbeat.cancelBudgetScopeWork(scope);
|
||||
} }).deliverPendingEnforcement(f.company.id);
|
||||
try {
|
||||
expect((await snapshot).enforcement).toMatchObject({ policyId: expect.any(String), version: expect.any(Number) });
|
||||
await budgets.upsertPolicy(f.company.id, { ...policyInput, amount: 100 }, "board");
|
||||
await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
await reserveRunBudget(db, f.company.id, f.run.id, f.project.id);
|
||||
} finally { release(); }
|
||||
await delivery;
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]).toMatchObject({ status: "running", resultJson: null });
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0].status).toBe("queued");
|
||||
});
|
||||
|
||||
it("keeps a durably claimed budget stop fenced after a later grant", async () => {
|
||||
const f = await fixture(), budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 1 }, "board");
|
||||
await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 2 });
|
||||
const [policy] = await db.select().from(budgetPolicies).where(eq(budgetPolicies.companyId, f.company.id));
|
||||
const scope: BudgetEnforcementScope = { companyId: f.company.id, scopeType: "company", scopeId: f.company.id,
|
||||
enforcement: { policyId: policy.id, version: policy.enforcementVersion } };
|
||||
expect(await withCurrentBudgetEnforcement(db, scope, async (tx) => {
|
||||
await tx.update(heartbeatRuns).set({ status: "running", resultJson: { cancellation: { reason: "Budget stop" } } }).where(eq(heartbeatRuns.id, f.run.id));
|
||||
return true;
|
||||
})).toBe(true);
|
||||
await budgets.upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 100 }, "board");
|
||||
await expect(reserveRunBudget(db, f.company.id, f.run.id, null)).rejects.toThrow("cancellation already requested");
|
||||
const effect = vi.fn(async () => true);
|
||||
for (const invalid of [
|
||||
scope,
|
||||
{ ...scope, enforcement: { ...scope.enforcement!, policyId: randomUUID() } },
|
||||
{ ...scope, scopeId: randomUUID() },
|
||||
{ ...scope, scopeType: "agent" as const },
|
||||
{ ...scope, enforcement: { ...scope.enforcement!, version: policy.enforcementVersion + 1 } },
|
||||
]) expect(await withCurrentBudgetEnforcement(db, invalid, effect)).toBeNull();
|
||||
expect(effect).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
});
|
||||
@@ -0,0 +1,91 @@
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { agents, companies, createDb } from "@paperclipai/db";
|
||||
import { costRoutes } from "../routes/costs.js";
|
||||
import { errorHandler } from "../middleware/error-handler.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("accounting operator authorization", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let companyId: string, foreignId: string, agentId: string;
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("accounting-authz-"); db = createDb(database.connectionString);
|
||||
const rows = await db.insert(companies).values([{ name: "A", issuePrefix: "AUTH_A" }, { name: "B", issuePrefix: "AUTH_B" }]).returning();
|
||||
companyId = rows[0].id; foreignId = rows[1].id;
|
||||
agentId = (await db.insert(agents).values({ companyId, name: "Worker", role: "engineer", adapterType: "process" }).returning())[0].id;
|
||||
},30_000);
|
||||
afterAll(async () => { await database?.cleanup(); });
|
||||
function app(actor: Record<string, unknown>) {
|
||||
const app = express(); app.use(express.json()); app.use((req,_res,next) => { req.actor = actor as typeof req.actor; next(); });
|
||||
app.use("/api", costRoutes(db)); app.use(errorHandler); return app;
|
||||
}
|
||||
const operations = [
|
||||
["get", "health", undefined], ["get", "inspect", undefined], ["get", "invoices", undefined],
|
||||
["get", `invoices/${randomUUID()}`, undefined], ["get", `events/${randomUUID()}/adjustments`, undefined],
|
||||
["post", "repair", { fingerprint: "a".repeat(64), reason: "test" }],
|
||||
["post", "provider-costs/import", {}], ["post", "retry", { runId: randomUUID() }], ["post", "invoices", {}], ["post", `events/${randomUUID()}/adjustments`, {}],
|
||||
] as const;
|
||||
it.each(operations)("denies agents for %s %s before reading or validating operator payloads", async (method, route, body) => {
|
||||
const response = await request(app({ type: "agent", companyId, agentId }))[method](`/api/companies/${companyId}/accounting/${route}`).send(body);
|
||||
expect(response.status).toBe(403);
|
||||
});
|
||||
it.each(operations)("denies nonmembers for %s %s", async (method, route, body) => {
|
||||
const response = await request(app({ type: "board", source: "session", userId: "operator", companyIds: [foreignId] }))[method](`/api/companies/${companyId}/accounting/${route}`).send(body);
|
||||
expect(response.status).toBe(403);
|
||||
});
|
||||
it("allows operator inspection and rejects viewer mutations", async () => {
|
||||
const board = { type: "board", source: "session", userId: "operator", companyIds: [companyId], memberships: [{ companyId, status: "active", membershipRole: "viewer" }] };
|
||||
const response = await request(app(board)).get(`/api/companies/${companyId}/accounting/inspect`);
|
||||
expect(response.status).toBe(200); expect(response.body.findings).toEqual([]);
|
||||
expect((await request(app(board)).post(`/api/companies/${companyId}/accounting/repair`).send({ fingerprint: response.body.fingerprint, reason: "test" })).status).toBe(403);
|
||||
});
|
||||
it.each(["member", "viewer", "unknown", undefined])("denies provider credential use by %s before payload validation", async (membershipRole) => {
|
||||
const actor = { type: "board", source: "session", userId: "member", companyIds: [companyId], memberships: [{ companyId, status: "active", membershipRole }] };
|
||||
const response = await request(app(actor)).post(`/api/companies/${companyId}/accounting/provider-costs/import`).send({});
|
||||
expect(response.status).toBe(403);
|
||||
});
|
||||
it.each([
|
||||
{ source: "session", memberships: "owner" },
|
||||
{ source: "session", memberships: "admin" },
|
||||
{ source: "session", isInstanceAdmin: true },
|
||||
{ source: "local_implicit" },
|
||||
])("allows billing operators through to payload validation: %j", async (input) => {
|
||||
const actor = { ...input, type: "board", userId: "operator", companyIds: [companyId], memberships: input.memberships ? [{ companyId, status: "active", membershipRole: input.memberships }] : [] };
|
||||
const response = await request(app(actor)).post(`/api/companies/${companyId}/accounting/provider-costs/import`).send({});
|
||||
expect(response.status).toBe(400);
|
||||
});
|
||||
it("does not accept an inactive admin or an admin role from another company", async () => {
|
||||
for (const membership of [
|
||||
{ companyId, status: "suspended", membershipRole: "admin" },
|
||||
{ companyId: foreignId, status: "active", membershipRole: "admin" },
|
||||
]) {
|
||||
const response = await request(app({ type: "board", source: "session", userId: "operator", companyIds: [companyId], memberships: [membership] }))
|
||||
.post(`/api/companies/${companyId}/accounting/provider-costs/import`).send({});
|
||||
expect(response.status).toBe(403);
|
||||
}
|
||||
});
|
||||
it("authorizes the user cost report and validates its date range", async () => {
|
||||
const path = `/api/companies/${companyId}/costs/by-user`;
|
||||
const board = app({ type: "board", source: "local_implicit", userId: "operator" });
|
||||
const response = await request(board).get(`${path}?period=all`);
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body).toEqual({ activeUserCount: 0, rows: [] });
|
||||
for (const query of ["from=invalid", "from=2026-02-01&to=2026-01-01", "period=all&from=2026-01-01"]) {
|
||||
expect((await request(board).get(`${path}?${query}`)).status).toBe(400);
|
||||
}
|
||||
for (const actor of [
|
||||
{ type: "board", source: "session", userId: "outsider", companyIds: [foreignId] },
|
||||
{ type: "agent", companyId: foreignId, agentId },
|
||||
]) expect((await request(app(actor)).get(path)).status).toBe(403);
|
||||
expect((await request(app({ type: "none" })).get(path)).status).toBe(401);
|
||||
});
|
||||
|
||||
it("validates correction and repair payloads before mutating", async () => {
|
||||
const board = app({ type: "board", source: "local_implicit", userId: "operator" });
|
||||
expect((await request(board).post(`/api/companies/${companyId}/accounting/repair`).send({ fingerprint: "old", reason: "" })).status).toBe(400);
|
||||
expect((await request(board).post(`/api/companies/${companyId}/accounting/events/${randomUUID()}/adjustments`).send({ idempotencyKey: "x", expectedCents: "0", correctedCents: "-0.000000001", reason: "Invalid", pricing: { source: "operator" } })).status).toBe(400);
|
||||
});
|
||||
});
|
||||
@@ -944,6 +944,25 @@ describe("AgentMail durable email pipeline", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["1999999999.9999999", true],
|
||||
["2000000000.0000000", false],
|
||||
["2000000000.0000001", false],
|
||||
])("compares exact monthly email spend %s at the budget boundary", async (spent, allowed) => {
|
||||
const f = await fixture();
|
||||
await f.receive(f.message());
|
||||
const [conversation] = await db.select().from(chatConversations).where(eq(chatConversations.companyId, f.companyId));
|
||||
const runId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, status: "running", runtimeMode: "native", nativeIssueId: conversation.issueId, contextSnapshot: { issueId: conversation.issueId } });
|
||||
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, conversation.issueId));
|
||||
await db.update(agents).set({ budgetMonthlyCents: 2000000000, spentMonthlyCents: sql`${spent}::numeric` }).where(eq(agents.id, f.agentId));
|
||||
const request = emailSendSchema.parse({ endpointId: f.endpointId, conversationId: conversation.id, replyToMessageId: [...f.messages.keys()][0], text: "Exact budget reply", idempotencyKey: randomUUID() });
|
||||
const queued = f.service.queueSend(f.companyId, request, { agentId: f.agentId, runId });
|
||||
if (allowed) expect((await queued).outcome).toBe("queued");
|
||||
else await expect(queued).rejects.toThrow("Agent is not available to send email");
|
||||
expect(f.sends).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("enforces one inbox owner across companies and reconnects the same identity", async () => {
|
||||
const f = await fixture();
|
||||
const other = await fixture();
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
// Dedicated real-process fault fixture. Only invoked with a throwaway test DB.
|
||||
import { createDb } from "@paperclipai/db";
|
||||
import { costService } from "../../services/costs.js";
|
||||
import { createRunUsageRecorder } from "../../services/usage-receipts.js";
|
||||
const [mode, encoded] = process.argv.slice(2);
|
||||
const input = JSON.parse(encoded);
|
||||
if (!process.env.PAPERCLIP_ACCOUNTING_TEST_DATABASE) throw new Error("Explicit test database required");
|
||||
const db = createDb(process.env.PAPERCLIP_ACCOUNTING_TEST_DATABASE);
|
||||
try {
|
||||
if (mode === "receipt") {
|
||||
const recorder = await createRunUsageRecorder(db, { companyId: input.companyId, runId: input.runId, adapterType: "process" }, input.directory);
|
||||
// The receipt is fsynced before persistUsageReceipt enters a transaction.
|
||||
// Freeze at this boundary, so the parent can SIGKILL us before any DB row.
|
||||
db.transaction = async () => { process.stdout.write("RECEIPT_DURABLE\n"); return new Promise(() => {}); };
|
||||
await recorder.capture({ costUsdExact: "0.012345678", usage: { inputTokens: 7, cachedInputTokens: 11, outputTokens: 3 }, complete: true, billingType: "metered_api" });
|
||||
} else {
|
||||
await Promise.all(Array.from({ length: 4 }, () => costService(db).createEvent(input.companyId, { ...input.receipt, occurredAt: new Date(input.receipt.occurredAt) })));
|
||||
process.stdout.write("WRITES_COMMITTED\n");
|
||||
}
|
||||
} finally { await db.$client.end(); }
|
||||
@@ -1,5 +1,6 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
normalizeAdapterRunUsage,
|
||||
resolveCacheAdjustedCostUsd,
|
||||
resolveLedgerCostStatus,
|
||||
} from "../services/heartbeat.js";
|
||||
@@ -35,6 +36,13 @@ describe("heartbeat cost accounting", () => {
|
||||
})).toBe("unpriced");
|
||||
});
|
||||
|
||||
it("distinguishes missing receipts, reported zeroes, and included subscription usage", () => {
|
||||
const noTokens = { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0 };
|
||||
expect(resolveLedgerCostStatus({ ...noTokens, costUsd: null })).toBe("unpriced");
|
||||
expect(resolveLedgerCostStatus({ ...noTokens, costUsd: 0 })).toBe("reported");
|
||||
expect(resolveLedgerCostStatus({ ...noTokens, costUsd: null, billingType: "subscription_included" })).toBe("reported");
|
||||
});
|
||||
|
||||
it("marks reported CLI cost as priced", () => {
|
||||
expect(resolveLedgerCostStatus({
|
||||
costUsd: 1.25,
|
||||
@@ -86,3 +94,17 @@ describe("heartbeat cost accounting", () => {
|
||||
})).toBe(1.5);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe("adapter usage basis", () => {
|
||||
const prior = { inputTokens: 100, cachedInputTokens: 500, outputTokens: 20 };
|
||||
it.each([undefined, null, "per_run"] as const)("keeps equal consecutive run usage when the basis is %s", (basis) => {
|
||||
expect(normalizeAdapterRunUsage(prior, prior, basis)).toEqual(prior);
|
||||
});
|
||||
it("subtracts session totals only when explicitly declared", () => {
|
||||
expect(normalizeAdapterRunUsage({ inputTokens: 110, cachedInputTokens: 700, outputTokens: 25 }, prior, "session_cumulative"))
|
||||
.toEqual({ inputTokens: 10, cachedInputTokens: 200, outputTokens: 5 });
|
||||
expect(normalizeAdapterRunUsage({ inputTokens: 5, cachedInputTokens: 10, outputTokens: 2 }, prior, "session_cumulative"))
|
||||
.toEqual({ inputTokens: 5, cachedInputTokens: 10, outputTokens: 2 });
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
agentRuntimeState,
|
||||
companies,
|
||||
costEvents,
|
||||
budgetReservations,
|
||||
createDb,
|
||||
documentRevisions,
|
||||
documents,
|
||||
@@ -827,6 +828,9 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
afterContinuationDispatchCheck = async ({ runId: guardedRunId, issueId: guardedIssueId }) => {
|
||||
expect(guardedRunId).toBe(runId);
|
||||
expect(guardedIssueId).toBe(issueId);
|
||||
// Budget admission must finish before this last ownership check. Putting
|
||||
// an awaited reservation inside dispatch lets work escape the row lock.
|
||||
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, runId)))[0]?.state).toBe("held");
|
||||
ordering.push("validated");
|
||||
parkPromise = Promise.resolve(
|
||||
db
|
||||
@@ -870,6 +874,8 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
expect(issue?.status).toBe("backlog");
|
||||
expect(ordering).toEqual(["validated", "parked"]);
|
||||
expect(countExecuteCallsForRun(runId)).toBe(0);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, runId)))[0]?.state).toBe("released");
|
||||
});
|
||||
|
||||
it("rate-limits skipped generic timer wakes by advancing the timer baseline", async () => {
|
||||
@@ -1311,6 +1317,14 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["1999999999.9999999", "2000000000.0000000"])("compares exact daily spend before admitting work (%s)", async costCents => {
|
||||
const { companyId, agentId } = await seedCompanyAndAgent({ heartbeatConfig: { maxDailyCostCents: 2_000_000_000 } });
|
||||
await db.insert(costEvents).values({ companyId, agentId, provider: "test", biller: "test", billingType: "metered_api", model: "test", costCents: costCents as unknown as number, occurredAt: new Date() });
|
||||
const run = await heartbeat.wakeup(agentId, { source: "on_demand", triggerDetail: "manual" });
|
||||
if (costCents === "1999999999.9999999") expect(run).not.toBeNull();
|
||||
else { expect(run).toBeNull(); expect(mockAdapterExecute).not.toHaveBeenCalled(); }
|
||||
});
|
||||
|
||||
it("treats zero daily cost cap as a hard stop", async () => {
|
||||
const { agentId } = await seedCompanyAndAgent({
|
||||
heartbeatConfig: {
|
||||
|
||||
@@ -796,6 +796,36 @@ describe("openapi routes", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("documents exact money, reviewed correction inputs, and board-only accounting access", () => {
|
||||
const { spec } = loadSpecRoutes();
|
||||
const base = "/api/companies/{companyId}/accounting/";
|
||||
for (const [suffix, methods] of [
|
||||
["health", ["get"]], ["inspect", ["get"]], ["repair", ["post"]], ["retry", ["post"]],
|
||||
["invoices", ["get", "post"]], ["invoices/{invoiceId}", ["get"]], ["events/{eventId}/adjustments", ["get", "post"]],
|
||||
] as const) {
|
||||
for (const method of methods) {
|
||||
const operation = spec.paths[`${base}${suffix}`][method];
|
||||
expect(operation["x-paperclip-authorization"]).toEqual({ actor: "board" });
|
||||
expect(operation.security).toEqual([{ BoardSessionAuth: [] }, { BoardApiKeyAuth: [] }]);
|
||||
expect(operation.responses["403"]).toBeDefined();
|
||||
}
|
||||
}
|
||||
const adjustment = spec.paths[`${base}events/{eventId}/adjustments`].post;
|
||||
expect(adjustment.responses["201"]).toBeDefined();
|
||||
expect(adjustment.responses["409"]).toBeDefined();
|
||||
expect(adjustment.responses["422"]).toBeDefined();
|
||||
expect(adjustment.requestBody.content["application/json"].schema.required).toEqual(expect.arrayContaining([
|
||||
"idempotencyKey", "expectedCents", "correctedCents", "reason", "pricing",
|
||||
]));
|
||||
for (const [endpoint, field] of [["cost-events", "costCents"], ["finance-events", "amountCents"]]) {
|
||||
const operation = spec.paths[`/api/companies/{companyId}/${endpoint}`].post;
|
||||
expect(operation.requestBody.content["application/json"].schema.properties[field]).toMatchObject({ oneOf: [{ type: "string" }, { type: "number" }] });
|
||||
expect(operation.responses["409"]).toBeDefined();
|
||||
}
|
||||
const parameters = spec.paths["/api/companies/{companyId}/costs/summary"].get.parameters;
|
||||
expect(parameters).toEqual(expect.arrayContaining([expect.objectContaining({ name: "period", in: "query", schema: expect.objectContaining({ enum: ["month", "all"] }) })]));
|
||||
});
|
||||
|
||||
it("documents board-only repository discovery and selection", () => {
|
||||
const { spec } = loadSpecRoutes();
|
||||
const discovery = spec.paths["/api/companies/{companyId}/project-repositories"].get;
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { fetchProviderDailyCosts } from "../services/provider-billing-import.js";
|
||||
import type { ImportProviderCosts } from "@paperclipai/shared";
|
||||
|
||||
const input: ImportProviderCosts = {
|
||||
provider: "openai",
|
||||
secretId: "00000000-0000-4000-8000-000000000001",
|
||||
accountId: "org_test",
|
||||
scopeIds: ["proj_test"],
|
||||
from: "2026-09-01",
|
||||
to: "2026-09-02",
|
||||
};
|
||||
const start = Date.parse(input.from) / 1000;
|
||||
function page(value = 0.06) {
|
||||
return {
|
||||
data: [
|
||||
{
|
||||
start_time: start,
|
||||
end_time: start + 86400,
|
||||
results: [
|
||||
{ project_id: "proj_test", amount: { currency: "usd", value } },
|
||||
],
|
||||
},
|
||||
],
|
||||
has_more: false,
|
||||
next_page: null,
|
||||
};
|
||||
}
|
||||
const response = (body: unknown) =>
|
||||
new Response(JSON.stringify(body), { status: 200 });
|
||||
describe("provider cost report ingestion", () => {
|
||||
it("uses fixed endpoints, admin identity and scoped daily amounts in the right currency units", async () => {
|
||||
const fetcher = vi.fn(async () => response(page()));
|
||||
expect(
|
||||
await fetchProviderDailyCosts(input, "private-admin", fetcher),
|
||||
).toEqual([
|
||||
{ day: "2026-09-01", scopeId: "proj_test", amountCents: "6.0000000" },
|
||||
]);
|
||||
const [url, options] = fetcher.mock.calls[0] as unknown as [
|
||||
URL,
|
||||
RequestInit,
|
||||
];
|
||||
expect(url.origin).toBe("https://api.openai.com");
|
||||
expect(url.searchParams.get("group_by[]")).toBe("project_id");
|
||||
expect(options.redirect).toBe("error");
|
||||
expect(options.headers).toMatchObject({
|
||||
"OpenAI-Organization": "org_test",
|
||||
});
|
||||
});
|
||||
it("verifies Anthropic organization identity and treats its amount as cents", async () => {
|
||||
const fetcher = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(response({ id: "org_test" }))
|
||||
.mockResolvedValueOnce(
|
||||
response({
|
||||
data: [
|
||||
{
|
||||
starting_at: "2026-09-01T00:00:00Z",
|
||||
ending_at: "2026-09-02T00:00:00Z",
|
||||
results: [
|
||||
{
|
||||
workspace_id: "proj_test",
|
||||
amount: "123.78912",
|
||||
currency: "USD",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
has_more: false,
|
||||
}),
|
||||
);
|
||||
expect(
|
||||
(
|
||||
await fetchProviderDailyCosts(
|
||||
{ ...input, provider: "anthropic" },
|
||||
"private-admin",
|
||||
fetcher,
|
||||
)
|
||||
)[0].amountCents,
|
||||
).toBe("123.7891200");
|
||||
await expect(
|
||||
fetchProviderDailyCosts(
|
||||
{ ...input, provider: "anthropic" },
|
||||
"private-admin",
|
||||
vi.fn(async () => response({ id: "other" })),
|
||||
),
|
||||
).rejects.toThrow("different provider organization");
|
||||
});
|
||||
it("excludes unrelated projects and supports an explicit empty daily bucket", async () => {
|
||||
const body = page();
|
||||
body.data[0].results[0].project_id = "other-company-project";
|
||||
expect(
|
||||
(
|
||||
await fetchProviderDailyCosts(
|
||||
input,
|
||||
"private",
|
||||
vi.fn(async () => response(body)),
|
||||
)
|
||||
)[0].amountCents,
|
||||
).toBe("0.0000000");
|
||||
});
|
||||
it.each([
|
||||
"missing-day",
|
||||
"missing-group",
|
||||
"overlap",
|
||||
"currency",
|
||||
"negative",
|
||||
"period",
|
||||
"cursor",
|
||||
"shape",
|
||||
])(
|
||||
"rejects %s rather than importing partial or invented totals",
|
||||
async (kind) => {
|
||||
const body: any = page();
|
||||
if (kind === "missing-day") body.data = [];
|
||||
if (kind === "missing-group") delete body.data[0].results[0].project_id;
|
||||
if (kind === "overlap") body.data.push(body.data[0]);
|
||||
if (kind === "currency") body.data[0].results[0].amount.currency = "EUR";
|
||||
if (kind === "negative") body.data[0].results[0].amount.value = -1;
|
||||
if (kind === "period") body.data[0].end_time++;
|
||||
if (kind === "cursor") body.has_more = true;
|
||||
if (kind === "shape") body.data[0].results[0].amount = {};
|
||||
await expect(
|
||||
fetchProviderDailyCosts(
|
||||
input,
|
||||
"private",
|
||||
vi.fn(async () => response(body)),
|
||||
),
|
||||
).rejects.toThrow();
|
||||
},
|
||||
);
|
||||
it("does not expose upstream errors or credentials", async () => {
|
||||
await expect(
|
||||
fetchProviderDailyCosts(
|
||||
input,
|
||||
"private-secret",
|
||||
vi.fn(
|
||||
async () =>
|
||||
new Response("private-secret raw diagnostics", { status: 403 }),
|
||||
),
|
||||
),
|
||||
).rejects.toThrow("Check the admin credential");
|
||||
});
|
||||
it("bounds response size and does not follow redirects", async () => {
|
||||
await expect(
|
||||
fetchProviderDailyCosts(
|
||||
input,
|
||||
"private",
|
||||
vi.fn(async () => new Response("x".repeat(2_000_001))),
|
||||
),
|
||||
).rejects.toThrow("too large");
|
||||
});
|
||||
it("collects all pages before returning and rejects a repeated cursor", async () => {
|
||||
const first = { ...page(), has_more: true, next_page: "next" };
|
||||
const second = page(0.01);
|
||||
second.data[0].start_time += 86400;
|
||||
second.data[0].end_time += 86400;
|
||||
const fetcher = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(response(first))
|
||||
.mockResolvedValueOnce(response(second));
|
||||
expect(
|
||||
await fetchProviderDailyCosts(
|
||||
{ ...input, to: "2026-09-03" },
|
||||
"private",
|
||||
fetcher,
|
||||
),
|
||||
).toHaveLength(2);
|
||||
const empty = { data: [], has_more: true, next_page: "same" };
|
||||
await expect(
|
||||
fetchProviderDailyCosts(
|
||||
input,
|
||||
"private",
|
||||
vi.fn(async () => response(empty)),
|
||||
),
|
||||
).rejects.toThrow("cursor");
|
||||
});
|
||||
it("preserves a valid report if closing its already-consumed response stream fails", async () => {
|
||||
let sent = false;
|
||||
const reader = {
|
||||
read: async () =>
|
||||
sent
|
||||
? { done: true }
|
||||
: ((sent = true),
|
||||
{ done: false, value: Buffer.from(JSON.stringify(page())) }),
|
||||
cancel: async () => {
|
||||
throw new Error("closed transport");
|
||||
},
|
||||
};
|
||||
const fetcher = vi
|
||||
.fn<typeof fetch>()
|
||||
.mockResolvedValue({
|
||||
ok: true,
|
||||
body: { getReader: () => reader },
|
||||
} as unknown as Response);
|
||||
expect(
|
||||
(await fetchProviderDailyCosts(input, "private", fetcher))[0].amountCents,
|
||||
).toBe("6.0000000");
|
||||
});
|
||||
it("includes unallocated costs only when the default scope was explicitly selected", async () => {
|
||||
const body = page();
|
||||
(body.data[0].results[0] as { project_id: string | null }).project_id =
|
||||
null;
|
||||
expect(
|
||||
await fetchProviderDailyCosts(
|
||||
{ ...input, scopeIds: ["default"] },
|
||||
"private",
|
||||
vi.fn(async () => response(body)),
|
||||
),
|
||||
).toEqual([
|
||||
{ day: "2026-09-01", scopeId: "default", amountCents: "6.0000000" },
|
||||
]);
|
||||
});
|
||||
it("rejects a provider amount with the other provider's unit/shape and missing period boundaries", async () => {
|
||||
const wrong: any = page();
|
||||
wrong.data[0].results[0].amount = "6";
|
||||
wrong.data[0].results[0].currency = "USD";
|
||||
await expect(
|
||||
fetchProviderDailyCosts(
|
||||
input,
|
||||
"private",
|
||||
vi.fn(async () => response(wrong)),
|
||||
),
|
||||
).rejects.toThrow("invalid billing amount");
|
||||
const missing: any = page();
|
||||
delete missing.data[0].start_time;
|
||||
delete missing.data[0].end_time;
|
||||
await expect(
|
||||
fetchProviderDailyCosts(
|
||||
input,
|
||||
"private",
|
||||
vi.fn(async () => response(missing)),
|
||||
),
|
||||
).rejects.toThrow("unexpected billing period");
|
||||
});
|
||||
});
|
||||
@@ -1078,7 +1078,7 @@ describeEmbeddedPostgres("status card routes", () => {
|
||||
model: "gpt-5.4",
|
||||
inputTokens: 5200,
|
||||
outputTokens: 980,
|
||||
costCents: 2,
|
||||
costCents: 0.1234567,
|
||||
occurredAt: new Date(),
|
||||
});
|
||||
const summaryWrite = await request(writerApp).put(`/api/status-cards/${created.body.id}/summary`).send({
|
||||
@@ -1111,7 +1111,7 @@ describeEmbeddedPostgres("status card routes", () => {
|
||||
summaryBody: "**Decide:** unblock launch approval.\n\n**Recent work:** launch review is waiting.",
|
||||
watchedIssueCount: 1,
|
||||
todayTokens: 6180,
|
||||
todayCostCents: 2,
|
||||
todayCostCents: 0.1234567,
|
||||
};
|
||||
const detail = await request(boardApp).get(`/api/status-cards/${created.body.id}`);
|
||||
expect(detail.status).toBe(200);
|
||||
@@ -1143,7 +1143,7 @@ describeEmbeddedPostgres("status card routes", () => {
|
||||
model: "gpt-5.4",
|
||||
inputTokens: 1300,
|
||||
outputTokens: 410,
|
||||
costCents: 1,
|
||||
costCents: 3_000_000_000.25,
|
||||
occurredAt: new Date(),
|
||||
});
|
||||
const incrementalWrite = await request(createApp(db, agentActor(company.id, summarizer.id, updateRun.id)))
|
||||
@@ -1155,7 +1155,7 @@ describeEmbeddedPostgres("status card routes", () => {
|
||||
model: "gpt-5.4",
|
||||
});
|
||||
expect(incrementalWrite.status).toBe(200);
|
||||
expect(await db.select().from(statusCardUpdates).then((rows) => rows.find((row) => row.kind === "incremental"))).toMatchObject({ inputTokens: 1300, outputTokens: 410 });
|
||||
expect(await db.select().from(statusCardUpdates).then((rows) => rows.find((row) => row.kind === "incremental"))).toMatchObject({ inputTokens: 1300, outputTokens: 410, costCents: 3_000_000_000.25 });
|
||||
const revisions = await request(boardApp).get(`/api/status-cards/${created.body.id}/summary-revisions`);
|
||||
expect(revisions.status).toBe(200);
|
||||
expect(revisions.body.map((row: { revisionNumber: number }) => row.revisionNumber)).toEqual([2, 1]);
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "..",
|
||||
"noEmit": true
|
||||
},
|
||||
"include": [
|
||||
"src/__tests__/cost-accounting-*.test.ts",
|
||||
"src/__tests__/codex-pricing.test.ts",
|
||||
"src/__tests__/provider-billing-import.test.ts",
|
||||
"src/__tests__/company-quota-windows.test.ts",
|
||||
"src/__tests__/fixtures/accounting-writer.ts",
|
||||
"src/**/*.d.ts",
|
||||
"vitest.accounting-mutations.config.ts",
|
||||
"vitest.accounting.config.ts",
|
||||
"../tests/e2e/cost-accounting.spec.ts"
|
||||
],
|
||||
"exclude": []
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { defineConfig } from "vitest/config";
|
||||
import base from "./vitest.config.js";
|
||||
export const mutations = {
|
||||
deduplication: { file: "/services/costs.ts", from: "idempotencyKey: parsed.data.idempotencyKey ?? null,", to: "idempotencyKey: null,", test: "deduplicates a retried receipt" },
|
||||
company: { file: "/services/costs.ts", from: "const conditions: ReturnType<typeof eq>[] = [eq(costEvents.companyId, companyId)];", to: "const conditions: ReturnType<typeof eq>[] = [sql`true`];", test: "isolates company reporting" },
|
||||
projection: { file: "/services/costs.ts", from: "+ ${delta}::numeric", to: "+ 0::numeric", test: "conserves agent projections" },
|
||||
threshold: { file: "/services/budgets.ts", from: "compareCents(observed.totalExact, policy.amount) >= 0", to: "compareCents(observed.totalExact, policy.amount) > 0", test: "stops at the exact budget boundary" },
|
||||
};
|
||||
const name = process.env.PAPERCLIP_ACCOUNTING_MUTATION;
|
||||
if (name && !(name in mutations)) throw new Error(`Unknown accounting mutation: ${name}`);
|
||||
const mutation = name ? mutations[name as keyof typeof mutations] : null;
|
||||
export default defineConfig({
|
||||
...base, root: fileURLToPath(new URL(".", import.meta.url)),
|
||||
plugins: [...(base.plugins ?? []), {
|
||||
name: "accounting-mutation-test-only", enforce: "pre",
|
||||
transform(code, id) {
|
||||
if (!mutation || !id.split("?")[0].endsWith(mutation.file)) return;
|
||||
if (!code.includes(mutation.from)) throw new Error(`Mutation anchor missing: ${name}`);
|
||||
// Replace precisely one site in Vite's in-memory module. Workspace source
|
||||
// stays untouched, including while another test process is running.
|
||||
console.error(`ACCOUNTING_MUTATION_APPLIED ${name}`);
|
||||
return code.replace(mutation.from, mutation.to);
|
||||
},
|
||||
}],
|
||||
test: { ...base.test, include: ["src/__tests__/cost-accounting-mutation-targets.test.ts"], coverage: { enabled: false } },
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { defineConfig } from "vitest/config";
|
||||
import base from "./vitest.config.js";
|
||||
|
||||
export default defineConfig({
|
||||
...base,
|
||||
root: fileURLToPath(new URL(".", import.meta.url)),
|
||||
test: {
|
||||
...base.test,
|
||||
include: [
|
||||
"src/__tests__/cost-accounting-*.test.ts",
|
||||
"src/__tests__/budgets-service.test.ts",
|
||||
"src/__tests__/costs-service.test.ts",
|
||||
"src/__tests__/codex-pricing.test.ts",
|
||||
"src/__tests__/provider-billing-import.test.ts",
|
||||
],
|
||||
coverage: {
|
||||
enabled: true,
|
||||
provider: "v8",
|
||||
reportOnFailure: true,
|
||||
reportsDirectory: "../coverage/accounting",
|
||||
reporter: ["text", "json", "json-summary", "html"],
|
||||
// Every module must retain its own floor; a well-covered helper cannot
|
||||
// hide a regression in ingestion, recovery, finance, or enforcement.
|
||||
thresholds: { perFile: true, lines: 98, branches: 90, functions: 96, statements: 96 },
|
||||
include: [
|
||||
"src/services/accounting-transaction.ts",
|
||||
"src/services/accounting-integrity.ts",
|
||||
"src/services/billing-reconciliation.ts",
|
||||
"src/services/budget-reservations.ts",
|
||||
"src/services/usage-receipts.ts",
|
||||
"src/services/receipt-fingerprint.ts",
|
||||
"src/services/cost-date-range.ts",
|
||||
"src/services/run-cost-accounting.ts",
|
||||
"src/services/costs.ts",
|
||||
"src/services/finance.ts",
|
||||
"src/services/budgets.ts",
|
||||
"src/services/codex-pricing.ts",
|
||||
"src/services/provider-billing-import.ts",
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
Reference in new issue
Block a user