import { afterEach, expect, it, vi } from "vitest"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { EvalSpendState, openEvalCostLedger, readEvalCostLedger, type EvalCostObservation, type EvalSpendLimits } from "./cost-ledger.js"; const roots: string[] = []; afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); const limits = (overrides: Partial = {}): EvalSpendLimits => ({ campaignId: "pilot", maxUsd: "1", deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 2, ...overrides }); const identity = { executionId: "codex.local.fixture", attemptId: "attempt-1", phase: "setup" as const }; const observation = (overrides: Partial = {}): EvalCostObservation => ({ observationId: "receipt", requestId: "request", source: "provider", provider: "openai", model: "fixture", providerRequestId: "provider-request", currency: "USD", route: "direct", classification: "reported", costUsd: "0.000000123", final: true, ...overrides }); async function fixture(overrides: Partial = {}) { const root = await mkdtemp(path.join(os.tmpdir(), "eval-cost-ledger-")); roots.push(root); const file = path.join(root, "costs.jsonl"); return { file, ledger: await openEvalCostLedger(file, limits(overrides)) }; } it("persists a reservation before dispatch and counts setup spending with nanodollar precision", async () => { const { file, ledger } = await fixture(); try { await ledger.reserve("request", identity, "0.1"); const beforeDispatch = (await readEvalCostLedger(file)).state.snapshot(); expect(beforeDispatch).toMatchObject({ committedUsd: "0.100000000", heldUsd: "0.100000000", unknownRequests: 1 }); await ledger.observe(observation()); await ledger.finish("request", "succeeded"); expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.000000123", committedUsd: "0.000000123", heldUsd: "0.000000000", requests: [{ phase: "setup", receiptComplete: true }] }); } finally { await ledger.close(); } expect((await readEvalCostLedger(file)).state.snapshot().knownUsd).toBe("0.000000123"); }); it("serializes competing admissions so concurrent requests cannot oversubscribe a cap", async () => { const { ledger } = await fixture(); try { const admitted = await Promise.allSettled([ledger.reserve("one", identity, "0.6"), ledger.reserve("two", identity, "0.6")]); expect(admitted.map(value => value.status)).toEqual(["fulfilled", "rejected"]); expect(ledger.snapshot().committedUsd).toBe("0.600000000"); } finally { await ledger.close(); } }); it("durably caps attempts including zero-request failures and rejects unadmitted requests", async () => { const cap = limits({ maxAttemptsPerExecution: 2 }); const { file, ledger } = await fixture({ maxAttemptsPerExecution: 2 }); await expect(ledger.reserve("request", identity, "0.1")).rejects.toThrow("no admitted attempt"); await ledger.admitAttempt(identity.executionId, "attempt-1"); await ledger.close(); const resumed = await openEvalCostLedger(file, cap); try { await expect(resumed.admitAttempt(identity.executionId, "attempt-1")).rejects.toThrow("already admitted"); const competing = await Promise.allSettled([ resumed.admitAttempt(identity.executionId, "attempt-2"), resumed.admitAttempt(identity.executionId, "attempt-3"), ]); expect(competing.map(result => result.status)).toEqual(["fulfilled", "rejected"]); await resumed.reserve("request", { ...identity, attemptId: "attempt-2" }, "0.1"); await resumed.admitAttempt("another.cell", "another-attempt"); await expect(resumed.reserve("wrong-cell", { ...identity, attemptId: "another-attempt" }, "0.1")).rejects.toThrow("no admitted attempt"); expect((await readEvalCostLedger(file)).state.snapshot().admittedAttempts).toHaveLength(3); } finally { await resumed.close(); } await expect(openEvalCostLedger(file, limits({ maxAttemptsPerExecution: 3 }))).rejects.toThrow("limits changed"); }); it("holds unknown/cancelled charges after restart and counts failed retries independently", async () => { const { file, ledger } = await fixture(); await ledger.reserve("request", identity, "0.5"); await ledger.finish("request", "cancelled"); await ledger.close(); const resumed = await openEvalCostLedger(file, limits()); try { expect(resumed.snapshot()).toMatchObject({ inflight: 0, unknownRequests: 1, committedUsd: "0.500000000" }); await resumed.reserve("retry", { ...identity, attemptId: "attempt-2" }, "0.5"); await resumed.observe(observation({ observationId: "retry-cost", requestId: "retry", providerRequestId: "provider-retry", costUsd: "0.2" })); await resumed.finish("retry", "failed"); expect(resumed.snapshot()).toMatchObject({ knownUsd: "0.200000000", heldUsd: "0.500000000", committedUsd: "0.700000000" }); await resumed.observe(observation({ costUsd: "0.3" })); expect(resumed.snapshot()).toMatchObject({ knownUsd: "0.500000000", heldUsd: "0.000000000" }); } finally { await resumed.close(); } }); it("deduplicates provider/gateway/native observations and preserves invoice adjustments", async () => { const { ledger } = await fixture(); try { await ledger.reserve("request", identity, "1"); await ledger.observe(observation({ source: "gateway", classification: "estimated", pricingVersion: "2026-10-20", costUsd: "0.3" })); await ledger.observe(observation({ source: "gateway", classification: "estimated", pricingVersion: "2026-10-20", costUsd: "0.3" })); await ledger.observe(observation({ observationId: "native", source: "native_otlp", costUsd: "0.2" })); await ledger.observe(observation({ observationId: "invoice", source: "invoice", classification: "reconciled", costUsd: "0.15" })); await ledger.finish("request", "succeeded"); expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.150000000", breakdown: { estimated: "0.000000000", reported: "0.000000000", reconciled: "0.150000000" } }); await expect(ledger.observe(observation({ costUsd: "0.7" }))).rejects.toThrow("identity conflict"); } finally { await ledger.close(); } }); it("retains partial costs and contradictory receipts without freeing their reservations", async () => { const { ledger } = await fixture(); try { await ledger.reserve("request", identity, "0.5"); await ledger.observe(observation({ final: false, costUsd: "0.2" })); await ledger.finish("request", "failed"); expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.200000000", heldUsd: "0.300000000", unknownRequests: 1 }); await ledger.observe(observation({ observationId: "final1", costUsd: "0.3" })); await ledger.observe(observation({ observationId: "final2", costUsd: "0.4" })); expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.400000000", heldUsd: "0.100000000", conflictingRequests: 1 }); await expect(ledger.reserve("later", identity, "0.1")).rejects.toThrow("receipts conflict"); } finally { await ledger.close(); } }); it("rejects ambiguous request joins and stops on measured overshoot", async () => { const { ledger } = await fixture(); try { await ledger.reserve("request", identity, "0.1"); await ledger.reserve("another", identity, "0.1"); await ledger.observe(observation({ costUsd: "1.1" })); await expect(ledger.observe(observation({ observationId: "duplicate-request", requestId: "another" }))).rejects.toThrow("another reservation"); expect(ledger.snapshot().overBudget).toBe(true); await ledger.finish("request", "succeeded"); await expect(ledger.reserve("later", identity, "0.1")).rejects.toThrow("spending cap"); } finally { await ledger.close(); } }); it("enforces deadlines, request count, concurrency, and explicit pricing/usage invariants", async () => { const expired = await fixture({ deadlineAt: "2000-01-01T00:00:00Z" }); await expect(expired.ledger.reserve("request", identity, "0.1")).rejects.toThrow("deadline"); await expired.ledger.close(); const { ledger } = await fixture({ maxRequests: 2, maxConcurrent: 1 }); try { await ledger.reserve("request", identity, "0.1"); await expect(ledger.reserve("two", identity, "0.1")).rejects.toThrow("concurrency"); await expect(ledger.observe(observation({ classification: "estimated" }))).rejects.toThrow("pricing provenance"); await expect(ledger.observe(observation({ tokens: { output: 1, reasoning: 2 } }))).rejects.toThrow("token usage"); await ledger.finish("request", "failed"); await ledger.reserve("two", identity, "0.1"); await ledger.finish("two", "failed"); await expect(ledger.reserve("three", identity, "0.1")).rejects.toThrow("request cap"); } finally { await ledger.close(); } }); it("fails closed on concurrent writers, changed limits, incomplete writes, and tampering", async () => { const { file, ledger } = await fixture(); await expect(openEvalCostLedger(file, limits())).rejects.toThrow("writer lock"); await ledger.reserve("request", identity, "0.1"); await ledger.close(); await expect(openEvalCostLedger(file, limits({ maxUsd: "2" }))).rejects.toThrow("limits changed"); const contents = await readFile(file, "utf8"); await writeFile(file, contents.slice(0, -1)); await expect(readEvalCostLedger(file)).rejects.toThrow("incomplete"); await writeFile(file, contents.replace('"maxUsd":"1"', '"maxUsd":"2"')); await expect(readEvalCostLedger(file)).rejects.toThrow("integrity"); }); it("rejects accidental raw payload fields and accepts equivalent receipt key order", async () => { const { file, ledger } = await fixture(); try { await ledger.reserve("request", identity, "0.1"); await expect(ledger.observe({ ...observation(), rawResponse: "private response" } as EvalCostObservation)).rejects.toThrow("Unknown ledger field"); await ledger.observe(observation()); await ledger.observe(Object.fromEntries(Object.entries(observation()).reverse()) as unknown as EvalCostObservation); expect(await readFile(file, "utf8")).not.toContain("private response"); } finally { await ledger.close(); } }); it("does not let a terminal estimate erase a stronger partial provider charge", async () => { const { ledger } = await fixture(); try { await ledger.reserve("request", identity, "0.8"); await ledger.observe(observation({ observationId: "provider-partial", final: false, costUsd: "0.6" })); await ledger.observe(observation({ observationId: "gateway-final", source: "gateway", classification: "estimated", pricingVersion: "fixture", costUsd: "0.1" })); await ledger.finish("request", "succeeded"); expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.600000000", heldUsd: "0.200000000", unknownRequests: 1, breakdown: { reported: "0.600000000", estimated: "0.000000000" } }); await expect(ledger.reserve("next", identity, "0.3")).rejects.toThrow("spending cap"); await ledger.observe(observation({ observationId: "provider-final", costUsd: "0.65" })); expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.650000000", heldUsd: "0.000000000", unknownRequests: 0 }); } finally { await ledger.close(); } }); it("replays admissions when the deadline crosses between validation and journal writing", async () => { const admittedAt = Date.parse("2098-12-31T23:59:59.999Z"); vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(admittedAt); const { file, ledger } = await fixture(); const validate = EvalSpendState.prototype.validate; const validation = vi.spyOn(EvalSpendState.prototype, "validate").mockImplementation(function (this: EvalSpendState, event, now) { validate.call(this, event, now); vi.setSystemTime(admittedAt + 2); }); try { await ledger.reserve("request", identity, "0.1"); validation.mockRestore(); expect((await readEvalCostLedger(file)).state.snapshot().committedUsd).toBe("0.100000000"); const records = (await readFile(file, "utf8")).trim().split("\n").map(line => JSON.parse(line)); expect(records[1].at).toBe(new Date(admittedAt).toISOString()); } finally { validation.mockRestore(); vi.useRealTimers(); await ledger.close(); } });