import { expect, it } from "vitest"; import { estimateProviderCost, normalizeProviderUsage, ProviderUsageAccumulator, type EvalModelPrice } from "./provider-usage.js"; const price: EvalModelPrice = { provider: "openai", model: "fixture", version: "fixture-1", sourceUrl: "https://provider.test/prices", inputUsdPerMillion: "2.5", cachedReadUsdPerMillion: "0.25", outputUsdPerMillion: "10", cacheWrite30mUsdPerMillion: "2", cacheWrite12hUsdPerMillion: "2" }; it("excludes cached reads from OpenAI input and treats reasoning as a subset of output", () => { const usage = normalizeProviderUsage("openai", { id: "response", model: "fixture", usage: { input_tokens: 100, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 40 }, output_tokens: 20, output_tokens_details: { reasoning_tokens: 15 } } }); expect(usage).toMatchObject({ complete: true, tokens: { input: 60, cachedRead: 40, cachedWrite: 0, output: 20, reasoning: 15 } }); expect(estimateProviderCost(usage, price)).toBe("0.000360000"); }); it("never turns missing usage or incompatible model/pricing into a free charge", () => { const absent = normalizeProviderUsage("openai", { model: "fixture", usage: { input_tokens: 0, output_tokens: 0 } }); expect(absent.complete).toBe(false); expect(estimateProviderCost(absent, price)).toBeNull(); const zero = normalizeProviderUsage("openai", { model: "fixture", usage: { input_tokens: 0, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 0 }, output_tokens: 0 } }); expect(estimateProviderCost(zero, price)).toBe("0.000000000"); expect(estimateProviderCost(zero, { ...price, model: "other" })).toBeNull(); }); it("does not add Anthropic cumulative output counters and prices cache writes by TTL", () => { const stream = new ProviderUsageAccumulator("anthropic"); stream.observe({ type: "message_start", message: { id: "message", model: "fixture", usage: { input_tokens: 10, output_tokens: 1, cache_read_input_tokens: 20, cache_creation_input_tokens: 30, cache_creation: { ephemeral_5m_input_tokens: 20, ephemeral_1h_input_tokens: 10 } } } }); stream.observe({ type: "message_delta", usage: { output_tokens: 5 } }); stream.observe({ type: "message_delta", usage: { output_tokens: 8 } }); expect(stream.receipt().final).toBe(false); stream.observe({ type: "message_stop" }); expect(stream.receipt()).toMatchObject({ final: true, complete: true, tokens: { input: 10, cachedRead: 20, cachedWrite: 30, output: 8 } }); expect(estimateProviderCost(stream.receipt(), { ...price, provider: "anthropic", cacheWrite5mUsdPerMillion: "3.125", cacheWrite1hUsdPerMillion: "5" })).toBe("0.000222500"); }); it("retains missing cache TTL and malformed token subsets as unknown", () => { const write = normalizeProviderUsage("anthropic", { model: "fixture", usage: { input_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 4, output_tokens: 1 } }); expect(estimateProviderCost(write, { ...price, provider: "anthropic" })).toBeNull(); expect(normalizeProviderUsage("openai", { usage: { input_tokens: 1, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 2 }, output_tokens: 1 } }).complete).toBe(false); expect(normalizeProviderUsage("openai", { usage: { input_tokens: 1, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 0 }, output_tokens: 1, output_tokens_details: { reasoning_tokens: 2 } } }).complete).toBe(false); }); it("uses one terminal Responses usage receipt regardless of streamed text or repeat notifications", () => { const stream = new ProviderUsageAccumulator("openai"); stream.observe({ type: "response.output_text.delta", delta: "not a usage receipt" }); expect(stream.receipt().complete).toBe(false); const terminal = { type: "response.completed", response: { id: "response", model: "fixture", usage: { input_tokens: 1, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 0 }, output_tokens: 1 } } }; stream.observe(terminal); stream.observe(terminal); expect(stream.receipt()).toMatchObject({ final: true, complete: true, tokens: { input: 1, output: 1 } }); expect(estimateProviderCost(stream.receipt(), price)).toBe("0.000012500"); }); const modernPrice: EvalModelPrice = { ...price, inputUsdPerMillion: "4", cachedReadUsdPerMillion: "0.4", outputUsdPerMillion: "20", cacheWrite30mUsdPerMillion: "5", cacheWrite12hUsdPerMillion: "10", longContext: { aboveInputTokens: 272_000, rates: { inputUsdPerMillion: "8", cachedReadUsdPerMillion: "0.8", outputUsdPerMillion: "30", cacheWrite30mUsdPerMillion: "10", cacheWrite12hUsdPerMillion: "20" } } }; const modernUsage = (total = 12_000, read = 4000, shortWrite = 3000, longWrite = 4000) => ({ model: "fixture", usage: { input_tokens: total, input_tokens_details: { cached_tokens: read, cache_write_tokens: shortWrite + longWrite, cache_write_tokens_details: { ttl_30m_tokens: shortWrite, ttl_12h_tokens: longWrite } }, output_tokens: 100, } }); it("separates modern OpenAI ordinary input and mixed-TTL writes without adding the base rate twice", () => { const receipt = normalizeProviderUsage("openai", modernUsage()); expect(receipt).toMatchObject({ complete: true, tokens: { input: 1000, cachedRead: 4000, cachedWrite: 7000 }, cacheWrite30m: 3000, cacheWrite12h: 4000 }); expect(estimateProviderCost(receipt, modernPrice)).toBe("0.062600000"); }); it("applies long-context pricing to the entire request using total input including cache", () => { expect(estimateProviderCost(normalizeProviderUsage("openai", modernUsage(272_000, 270_000, 0, 1000)), modernPrice)).toBe("0.124000000"); expect(estimateProviderCost(normalizeProviderUsage("openai", modernUsage(272_001, 270_000, 0, 1000)), modernPrice)).toBe("0.247008000"); }); it("keeps missing modern write counters or missing/inconsistent TTL splits unknown", () => { const missing = modernUsage(); delete (missing.usage.input_tokens_details as any).cache_write_tokens; expect(normalizeProviderUsage("openai", missing).complete).toBe(false); expect(estimateProviderCost(normalizeProviderUsage("openai", missing), modernPrice)).toBeNull(); const noTtl = modernUsage(); delete (noTtl.usage.input_tokens_details as any).cache_write_tokens_details; expect(estimateProviderCost(normalizeProviderUsage("openai", noTtl), modernPrice)).toBeNull(); const mismatch = modernUsage(); mismatch.usage.input_tokens_details.cache_write_tokens_details.ttl_12h_tokens++; expect(normalizeProviderUsage("openai", mismatch).complete).toBe(false); }); const policyPrice = { ...modernPrice, model: "gpt-5.6-sol" }; function requestPolicyReceipt(request: Record = {}, details: Record = {}) { const stream = new ProviderUsageAccumulator("openai", { model: policyPrice.model, input: "hello", ...request }); stream.observe({ type: "response.completed", response: { model: policyPrice.model, usage: { input_tokens: 12_000, input_tokens_details: { cached_tokens: 4000, cache_write_tokens: 7000, ...details }, output_tokens: 100, } } }); return stream.receipt(); } it.each([ [{}, "30m", "0.042600000"], [{ prompt_cache_retention: "24h" }, "30m", "0.042600000"], [{ prompt_cache_options: { mode: "implicit", ttl: "12h" } }, "12h", "0.077600000"], ] as const)("prices omitted write TTLs from a documented visible request policy: %j", (request, ttl, cost) => { const receipt = requestPolicyReceipt(request); expect(receipt).toMatchObject({ final: true, complete: true, cacheWriteAttribution: { source: "request_policy", version: "openai-prompt-cache-2026-10-20", ttl, } }); expect(estimateProviderCost(receipt, policyPrice)).toBe(cost); }); it.each([ { model: "unqualified-model" }, { previous_response_id: "stored-response" }, { conversation: "stored-conversation" }, { prompt: { id: "stored-prompt" } }, { context_management: [] }, { input: [{ type: "item_reference", id: "stored-item" }] }, { input: [{ type: "compaction", encrypted_content: "opaque" }] }, { input: [{ role: "user", content: [{ type: "input_text", text: "hello", prompt_cache_breakpoint: { mode: "explicit", ttl: "12h" } }] }] }, { prompt_cache_options: { mode: "explicit" } }, { prompt_cache_options: { ttl: "unknown" } }, { prompt_cache_options: { ttl: null } }, { prompt_cache_options: null }, { prompt_cache_options: { future_option: true } }, ])("keeps hidden, explicit, or unqualified cache policies unknown: %j", request => { const receipt = requestPolicyReceipt(request); expect(receipt.cacheWriteAttribution).toBeUndefined(); expect(estimateProviderCost(receipt, policyPrice)).toBeNull(); }); it("prefers provider TTL splits and never repairs malformed or partial provider counters", () => { const reported = requestPolicyReceipt({}, { cache_write_tokens_details: { ttl_30m_tokens: 3000, ttl_12h_tokens: 4000 } }); expect(reported.cacheWriteAttribution).toBeUndefined(); expect(estimateProviderCost(reported, policyPrice)).toBe("0.062600000"); for (const details of [{ cache_write_tokens: undefined }, { cache_write_tokens_details: null }, { cache_write_tokens_details: { ttl_30m_tokens: 7000 } }, { cache_write_tokens_details: { ttl_30m_tokens: 7000, ttl_12h_tokens: 1 } }]) { const receipt = requestPolicyReceipt({}, details); expect(receipt.cacheWriteAttribution).toBeUndefined(); expect(estimateProviderCost(receipt, policyPrice)).toBeNull(); } }); it.each([ ["openai", { input_tokens: 1001, output_tokens: 1 }, 1001], ["openai", { input_tokens_details: { cached_tokens: 1001 } }, 1001], ["anthropic", { input_tokens: 1, cache_read_input_tokens: 2, cache_creation_input_tokens: 1000, cache_creation: { ephemeral_5m_input_tokens: 999, ephemeral_1h_input_tokens: 0 } }, 1003], ["anthropic", { input_tokens: 1, cache_creation: { ephemeral_5m_input_tokens: 900, ephemeral_1h_input_tokens: 100 } }, 1001], ] as const)("retains %s input bounds independently of incomplete or contradictory price counters", (provider, usage, input) => { const receipt = normalizeProviderUsage(provider, { usage }); expect(receipt).toMatchObject({ complete: false, tokenBounds: { input, valid: true } }); expect(estimateProviderCost(receipt, { ...price, provider })).toBeNull(); }); it.each([null, -1, 1.5, "1001", Number.MAX_SAFE_INTEGER + 1])("marks malformed input evidence %j ineligible for a bound", input_tokens => { expect(normalizeProviderUsage("openai", { usage: { input_tokens } }).tokenBounds?.valid).toBe(false); });