diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.md index 6f4a3a643e1..043097ac7fa 100644 --- a/tests/runner-e2e/OPERATIONS.md +++ b/tests/runner-e2e/OPERATIONS.md @@ -502,6 +502,62 @@ does not open a new spending window, change limits, dispatch tests or rewrite previous records. Regenerate the campaign report from the same ledger afterward; an old report snapshot is not rewritten by an offline receipt import. +### Compare retained Anthropic billing exports + +Use the read-only comparison command to repeat a provider-window comparison +without dispatching a test or changing the cost journal: + +```sh +node --import ./server/node_modules/tsx/dist/esm/index.mjs tests/runner-e2e/anthropic-billing-comparison.ts /private/path/comparison-config.json /private/path/new-comparison.json +``` + +The private configuration has this shape. `apiKeyLabel` is the label in the CSV, +not an API credential. Copy `sourceDataAsOf` from the provider's reporting view. +Use absolute input paths and retain the original exported CSV bytes. + +```json +{ + "version": 1, + "manifest": "/private/path/manifest.json", + "ledger": "/private/path/ledger.jsonl", + "usageExport": "/private/path/claude-usage.csv", + "costExport": "/private/path/claude-cost.csv", + "workspace": "Dedicated eval workspace", + "apiKeyLabel": "dedicated-eval-key", + "date": "2026-10-23", + "sourceDataAsOf": "2026-10-23T18:07:00Z" +} +``` + +The first supported export shape is one Claude Sonnet 5 campaign, one workspace +and key label, standard global inference, and context buckets at or below 200k. +Usage rows are hourly; cost rows are daily. Both must cover the selected UTC +date, and every selected journal request must have finished inside that day and +before the source timestamp. The command rejects mixed scopes, duplicate rows, +unsupported pricing dimensions, conflicting observations and damaged journals. +A provider export may still be delayed after these checks pass. Matching scope +labels do not establish exclusive use of a key or completeness of its export. + +The output records source hashes, the journal checkpoint, the reporting window, +its provisional status, provider tokens priced with the retained manifest, +rounded provider charges, and the retained request subtotal. Complete and +partial token receipts remain distinct from dollar-cost completeness. Each known +token category counts once per request even when the cost or other token fields +are unknown. Missing-field counts remain explicit. Final token evidence takes +precedence over partial evidence; equally strong contradictory values fail. Canceled requests without a +returned model identity remain explicitly unconfirmed. The token and estimated +USD differences can be positive or negative; neither is assigned to a request. +Non-model charges are shown separately. The report does not add the provider +total to the ledger subtotal or interpret rounded zero charges as free usage. + +The command creates a new mode-0600 output and refuses to replace an existing +file. It makes no network or provider call, does not import receipts, and leaves +all reservations and historical evidence intact. A comparison is supporting +evidence for reconciliation, not a settled invoice or an admission override. +Keep the config, raw exports and comparison private; they are not public report +projections. Other providers and export shapes still require their own reviewed +comparison path. + The checked pricecard expires on November 19, 2026. Admission stops when the pricecard expires, the catalog model changes, the project is missing, or the preflight already exists. Review current official prices and context limits diff --git a/tests/runner-e2e/anthropic-billing-comparison.test.ts b/tests/runner-e2e/anthropic-billing-comparison.test.ts new file mode 100644 index 00000000000..45856ae5773 --- /dev/null +++ b/tests/runner-e2e/anthropic-billing-comparison.test.ts @@ -0,0 +1,152 @@ +import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { billingCsv, compareAnthropicBilling, writeAnthropicBillingComparison, type AnthropicBillingComparisonConfig } from "./anthropic-billing-comparison.js"; +import { openEvalCostLedger } from "./cost-ledger.js"; + +const roots: string[] = []; +afterEach(async () => { vi.restoreAllMocks(); await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +const usageHeader = "usage_date_utc,model_version,api_key,workspace,usage_type,context_window,usage_input_tokens_no_cache,usage_input_tokens_cache_write_5m,usage_input_tokens_cache_write_1h,usage_input_tokens_cache_read,usage_output_tokens,web_search_count,inference_geo,speed\n"; +const usageRow = '2020-01-02 12:00,claude-sonnet-5,"key, label",workspace,standard,≤ 200k,20,100,0,30,8,0,global,standard\n'; +const costHeader = "usage_date_utc,model,workspace,api_key,usage_type,context_window,token_type,cost_usd,list_price_usd,cost_type,inference_geo,speed,api_key_id,api_key_status\n"; +const costRows = ["input_no_cache", "input_cache_read", "input_cache_write_5m", "input_cache_write_1h", "output"].map(token => + `2020-01-02,Claude Sonnet 5,workspace,"key, label",message,≤ 200k,${token},0.00,0.00,token,global,standard,key-id,\n`).join(""); + +async function fixture() { + const root = await mkdtemp(path.join(tmpdir(), "billing-comparison-")); roots.push(root); + const config: AnthropicBillingComparisonConfig = { version: 1, manifest: path.join(root, "manifest.json"), ledger: path.join(root, "ledger.jsonl"), + usageExport: path.join(root, "usage.csv"), costExport: path.join(root, "cost.csv"), workspace: "workspace", apiKeyLabel: "key, label", + date: "2020-01-02", sourceDataAsOf: "2020-01-03T01:00:00Z" }; + const limits = { campaignId: "comparison", maxUsd: "10", deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 1 }; + const clock = vi.spyOn(Date, "now").mockReturnValue(Date.parse("2020-01-02T12:05:00Z")); + const ledger = await openEvalCostLedger(config.ledger, limits); + const observation = { observationId: "provider-1", requestId: "complete", provider: "anthropic", model: "claude-sonnet-5", source: "provider" as const, + currency: "USD" as const, route: "braintrust" as const, classification: "estimated" as const, pricingVersion: "fixture", costUsd: "0.000326", + tokens: { input: 10, cachedWrite: 100, cachedRead: 30, output: 5 }, final: true }; + try { + await ledger.reserve("complete", { executionId: "cell", attemptId: "attempt", phase: "setup" }, "1"); + await ledger.observe(observation); + await ledger.observe({ ...observation, observationId: "gateway-1", source: "gateway" }); + await ledger.finish("complete", "succeeded"); + await ledger.reserve("cancelled", { executionId: "cell", attemptId: "attempt", phase: "agent" }, "1"); + await ledger.observe({ observationId: "cancelled-usage", requestId: "cancelled", provider: "anthropic", + source: "gateway", currency: "USD", route: "braintrust", classification: "unknown", final: false }); + await ledger.finish("cancelled", "cancelled"); + } finally { await ledger.close(); clock.mockRestore(); } + const price = { provider: "anthropic", model: "claude-sonnet-5", version: "fixture", sourceUrl: "https://example.test/prices", + inputUsdPerMillion: "2", cachedReadUsdPerMillion: "0.2", cacheWrite5mUsdPerMillion: "2.5", cacheWrite1hUsdPerMillion: "4", outputUsdPerMillion: "10" }; + await Promise.all([ + writeFile(config.manifest, JSON.stringify({ version: 1, limits, executionIds: ["cell"], models: [{ provider: "anthropic", price }] })), + writeFile(config.usageExport, usageHeader + usageRow), writeFile(config.costExport, costHeader + costRows), + ]); + return { config, root, limits, observation }; +} + +it("compares provider usage once, keeps the cancelled residual explicit, and preserves the journal", async () => { + const { config, root } = await fixture(), before = await readFile(config.ledger); + const configFile = path.join(root, "config.json"), output = path.join(root, "comparison.json"); + await writeFile(configFile, JSON.stringify(config)); + const result = await writeAnthropicBillingComparison(configFile, output); + expect(result).toMatchObject({ + provider: { tokenEstimateUsd: "0.000376000", roundedTokenCostUsd: "0.000000000", tokenEstimateConsistentWithRoundedCost: true }, + retained: { requests: 2, completeRequests: 1, missingTokenRequests: 1, partialTokenRequests: 0, unconfirmedModelRequests: 1, knownUsd: "0.000326000", + observedTokens: { input: 10, cachedRead: 30, cachedWrite: 100, output: 5 } }, + difference: { tokens: { input: 10, cachedRead: 0, cachedWrite: 0, output: 3 }, estimatedUsd: "0.000050000" }, + invoiceReconciled: false, requestCostsAllocated: false, holdsReleased: false, + }); + expect(result.sources).toHaveLength(4); + expect(result.sources.every(source => /^[a-f0-9]{64}$/.test(source.sha256))).toBe(true); + expect(await readFile(config.ledger)).toEqual(before); + expect((await stat(output)).mode & 0o777).toBe(0o600); + expect(await readFile(output, "utf8")).not.toContain("key, label"); + await expect(writeAnthropicBillingComparison(configFile, output)).rejects.toThrow("EEXIST"); + expect(await readFile(config.ledger)).toEqual(before); +}); + +it("retains partial tokens once and exposes a negative provider difference", async () => { + const { config, limits, observation } = await fixture(); + const ledger = await openEvalCostLedger(config.ledger, limits); + try { + await ledger.observe({ ...observation, observationId: "partial", requestId: "cancelled", final: false, + costUsd: "0.000100", tokens: { input: 40, cachedRead: 0, cachedWrite: 0, output: 2 } }); + } finally { await ledger.close(); } + expect(await compareAnthropicBilling(config)).toMatchObject({ retained: { partialTokenRequests: 1, missingTokenRequests: 0, knownUsd: "0.000426000" }, + difference: { tokens: { input: -30, output: 1 }, estimatedUsd: "-0.000050000" } }); +}); + +it.each([ + [{ input: 40 }, { input: 0, cachedRead: 1, cachedWrite: 1, output: 1 }, 5], + [{ input: 40, cachedRead: 0, cachedWrite: 0, output: 2 }, { input: 0, cachedRead: 0, cachedWrite: 0, output: 0 }, 7], +])("preserves known token fields on unpriced partial receipts", async (tokens, missingFields, expectedOutput) => { + const { config, limits } = await fixture(); + const ledger = await openEvalCostLedger(config.ledger, limits); + try { + await ledger.observe({ observationId: "unpriced-token-evidence", requestId: "cancelled", provider: "anthropic", source: "gateway", + currency: "USD", route: "braintrust", classification: "unknown", tokens, final: false }); + } finally { await ledger.close(); } + expect(await compareAnthropicBilling(config)).toMatchObject({ retained: { partialTokenRequests: 1, missingTokenRequests: 0, + knownUsd: "0.000326000", observedTokens: { input: 50, output: expectedOutput }, missingTokenFields: missingFields }, + difference: { tokens: { input: -30 } }, holdsReleased: false }); +}); + +it("prefers final token evidence while preserving an unknown dollar cost", async () => { + const { config, limits } = await fixture(); + const ledger = await openEvalCostLedger(config.ledger, limits); + const observation = { requestId: "cancelled", provider: "anthropic", model: "claude-sonnet-5", source: "gateway" as const, + currency: "USD" as const, route: "braintrust" as const, classification: "unknown" as const }; + try { + await ledger.observe({ ...observation, observationId: "earlier", tokens: { input: 40, output: 1 }, final: false }); + await ledger.observe({ ...observation, observationId: "final-unpriced", tokens: { input: 40, cachedRead: 0, cachedWrite: 0, output: 2 }, final: true }); + } finally { await ledger.close(); } + expect(await compareAnthropicBilling(config)).toMatchObject({ retained: { completeRequests: 1, completeTokenRequests: 2, + partialTokenRequests: 0, missingTokenRequests: 0, knownUsd: "0.000326000", observedTokens: { input: 50, output: 7 }, + missingTokenFields: { input: 0, cachedRead: 0, cachedWrite: 0, output: 0 } } }); +}); + +it("rejects duplicate token observations that disagree despite equal cost", async () => { + const { config, limits, observation } = await fixture(); + const ledger = await openEvalCostLedger(config.ledger, limits); + try { await ledger.observe({ ...observation, observationId: "contradiction", tokens: { ...observation.tokens, output: 6 } }); } + finally { await ledger.close(); } + await expect(compareAnthropicBilling(config)).rejects.toThrow("disagree on tokens"); +}); + +it.each([ + ["mixed workspace", usageRow.replace(",workspace,", ",other,"), "different workspace"], + ["batch price", usageRow.replace(",standard,≤", ",batch,≤"), "Unsupported usage"], + ["long context", usageRow.replace("≤ 200k", "> 200k"), "pricing dimensions"], + ["geography", usageRow.replace(",global,", ",us,"), "pricing dimensions"], + ["negative tokens", usageRow.replace(",20,100,", ",-20,100,"), "token count"], + ["duplicate row", usageRow + usageRow, "Duplicate usage"], + ["different date", usageRow.replace("2020-01-02", "2020-01-01"), "date"], +])("rejects %s instead of silently filtering the export", async (_name, rows, error) => { + const { config } = await fixture(); await writeFile(config.usageExport, usageHeader + rows); + await expect(compareAnthropicBilling(config)).rejects.toThrow(error); +}); + +it("requires complete cost categories and preserves separate non-model costs", async () => { + const { config } = await fixture(); + await writeFile(config.costExport, costHeader + costRows.split("\n").slice(0, 4).join("\n")); + await expect(compareAnthropicBilling(config)).rejects.toThrow("all five token categories"); + await writeFile(config.costExport, costHeader + costRows + '2020-01-02,--,workspace,"key, label",--,--,--,0.25,0.25,web_search,--,--,key-id,\n'); + expect(await compareAnthropicBilling(config)).toMatchObject({ provider: { roundedOtherCostUsd: "0.250000000", tokenEstimateUsd: "0.000376000" } }); +}); + +it("rejects a future snapshot, changed campaign identity and incomplete journal", async () => { + const { config } = await fixture(); + await expect(compareAnthropicBilling({ ...config, sourceDataAsOf: "2099-01-01T00:00:00Z" })).rejects.toThrow("reporting window"); + const manifest = JSON.parse(await readFile(config.manifest, "utf8")); + manifest.limits.campaignId = "other"; await writeFile(config.manifest, JSON.stringify(manifest)); + await expect(compareAnthropicBilling(config)).rejects.toThrow("campaign model"); + manifest.limits.campaignId = "comparison"; await writeFile(config.manifest, JSON.stringify(manifest)); + await writeFile(config.ledger, '{"partial":', { flag: "a" }); + const before = await readFile(config.ledger); + await expect(compareAnthropicBilling(config)).rejects.toThrow("durable record"); + expect(await readFile(config.ledger)).toEqual(before); +}); + +it("parses quoted commas, escaped quotes, newlines and CRLF while rejecting malformed CSV", () => { + expect(billingCsv('\uFEFFa,b\r\n"x,y","z""q"\r\n"line\nnext",end')).toEqual([{ a: "x,y", b: 'z"q' }, { a: "line\nnext", b: "end" }]); + for (const source of ['a,a\n1,2', 'a,b\n1', 'a,b\n"x"tail,y', 'a,b\n"unterminated,y']) expect(() => billingCsv(source)).toThrow(); +}); diff --git a/tests/runner-e2e/anthropic-billing-comparison.ts b/tests/runner-e2e/anthropic-billing-comparison.ts new file mode 100644 index 00000000000..bf4b20fd4bf --- /dev/null +++ b/tests/runner-e2e/anthropic-billing-comparison.ts @@ -0,0 +1,222 @@ +import { createHash } from "node:crypto"; +import { readFile, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { usdToUnits } from "../../packages/shared/src/money.js"; +import { readEvalCostLedger, validEvalLedgerId } from "./cost-ledger.js"; +import { estimateProviderCost, type EvalModelPrice } from "./provider-usage.js"; + +export interface AnthropicBillingComparisonConfig { + version: 1; + manifest: string; + ledger: string; + usageExport: string; + costExport: string; + workspace: string; + apiKeyLabel: string; + date: string; + sourceDataAsOf: string; +} +const hash = (value: string | Buffer) => createHash("sha256").update(value).digest("hex"); +const money = (value: bigint) => `${value < 0n ? "-" : ""}${(value < 0n ? -value : value) / 1_000_000_000n}.${String((value < 0n ? -value : value) % 1_000_000_000n).padStart(9, "0")}`; +const tokenKeys = ["input", "cachedRead", "cachedWrite", "output"] as const; +type Tokens = Record; +const emptyTokens = (): Tokens => ({ input: 0, cachedRead: 0, cachedWrite: 0, output: 0 }); +const addTokens = (target: Tokens, source: Tokens) => { + for (const key of tokenKeys) { + if (!Number.isSafeInteger(source[key]) || source[key] < 0 || !Number.isSafeInteger(target[key] + source[key])) throw new Error("Invalid or overflowing token count"); + target[key] += source[key]; + } +}; +const count = (value: string | undefined) => { + if (!value || !/^(?:0|[1-9][0-9]*)$/.test(value) || !Number.isSafeInteger(Number(value))) throw new Error("Invalid exported token count"); + return Number(value); +}; + +export function billingCsv(source: string): Record[] { + if (Buffer.byteLength(source) > 8 * 1024 * 1024) throw new Error("Billing CSV exceeds 8 MiB"); + const rows: string[][] = [], row: string[] = []; + let cell = "", quoted = false, closed = false; + const text = source.replace(/^\uFEFF/, ""); + for (let i = 0; i < text.length; i++) { + const c = text[i]!; + if (quoted) { + if (c === '"' && text[i + 1] === '"') { cell += '"'; i++; } + else if (c === '"') { quoted = false; closed = true; } + else cell += c; + } else if (c === ',' || c === '\n' || c === '\r') { + row.push(cell); cell = ""; closed = false; + if (c !== ',') { + if (c === '\r' && text[i + 1] === '\n') i++; + rows.push(row.splice(0)); + } + } else if (c === '"' && !cell && !closed) quoted = true; + else { + if (closed || c === '"') throw new Error("Malformed quoted billing CSV"); + cell += c; + } + } + if (quoted) throw new Error("Unterminated billing CSV field"); + if (cell || row.length || closed) { row.push(cell); rows.push(row); } + const headers = rows.shift(); + if (!headers?.length || headers.some(value => !value) || new Set(headers).size !== headers.length || !rows.length) throw new Error("Billing CSV requires unique headers and data"); + return rows.map(values => { + if (values.length !== headers.length) throw new Error("Billing CSV column count changed"); + return Object.fromEntries(headers.map((key, i) => [key, values[i]!])); + }); +} + +async function boundedFile(file: string): Promise { + const info = await stat(file); + if (!info.isFile() || info.size > 8 * 1024 * 1024) throw new Error("Comparison input must be a file of at most 8 MiB"); + const bytes = await readFile(file); + if (bytes.length > 8 * 1024 * 1024) throw new Error("Comparison input grew beyond 8 MiB"); + return bytes; +} + +export async function compareAnthropicBilling(config: AnthropicBillingComparisonConfig, now = Date.now()) { + if (!config || typeof config !== "object" || Array.isArray(config) || config.version !== 1 + || Object.keys(config).some(key => !["version", "manifest", "ledger", "usageExport", "costExport", "workspace", "apiKeyLabel", "date", "sourceDataAsOf"].includes(key)) + || [config.manifest, config.ledger, config.usageExport, config.costExport, config.workspace, config.apiKeyLabel].some(value => typeof value !== "string" || !value || value.length > 4096) + || !/^\d{4}-\d{2}-\d{2}$/.test(config.date) || !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/.test(config.sourceDataAsOf)) throw new Error("Invalid billing comparison config"); + const startedAt = Date.parse(`${config.date}T00:00:00Z`), asOf = Date.parse(config.sourceDataAsOf), endedAt = startedAt + 86_400_000; + if (!Number.isFinite(startedAt) || new Date(startedAt).toISOString().slice(0, 10) !== config.date + || !Number.isFinite(asOf) || asOf < startedAt || asOf > now) throw new Error("Invalid export reporting window"); + const [manifestBytes, ledgerBytes, usageBytes, costBytes] = await Promise.all([ + boundedFile(config.manifest), boundedFile(config.ledger), boundedFile(config.usageExport), boundedFile(config.costExport), + ]); + const manifest = JSON.parse(manifestBytes.toString("utf8")); + const loaded = await readEvalCostLedger(config.ledger), snapshot = loaded.state.snapshot(); + const lines = ledgerBytes.toString("utf8").trimEnd().split("\n").map(line => JSON.parse(line)); + if (lines.at(-1)?.hash !== loaded.previous || lines.length !== loaded.sequence) throw new Error("Ledger changed during comparison"); + if (manifest.version !== 1 || manifest.limits?.campaignId !== snapshot.campaignId || !Array.isArray(manifest.models) + || manifest.models.length !== 1 || manifest.models[0].provider !== "anthropic" + || !Array.isArray(manifest.executionIds) || !manifest.executionIds.length || manifest.executionIds.some((id: unknown) => !validEvalLedgerId(id))) throw new Error("Comparison requires one Anthropic campaign model"); + const price = manifest.models[0].price as EvalModelPrice; + if (price?.provider !== "anthropic" || price.model !== "claude-sonnet-5" || !validEvalLedgerId(price.version) || price.longContext) throw new Error("Unsupported Anthropic export price contract"); + const usage = billingCsv(usageBytes.toString("utf8")), costs = billingCsv(costBytes.toString("utf8")); + const scope = (row: Record) => { + if (row.workspace !== config.workspace || row.api_key !== config.apiKeyLabel) throw new Error("Export contains a different workspace or API key label"); + }; + const dimensions = (row: Record) => { + if (row.context_window !== "≤ 200k" || row.inference_geo !== "global" || row.speed !== "standard") throw new Error("Unsupported export pricing dimensions"); + }; + const providerTokens = emptyTokens(), seenUsage = new Set(), seenCosts = new Set(); + let cacheWrite5m = 0, cacheWrite1h = 0; + for (const row of usage) { + scope(row); dimensions(row); + if (row.model_version !== price.model || row.usage_type !== "standard" || count(row.web_search_count) !== 0 + || !new RegExp(`^${config.date} (?:[01][0-9]|2[0-3]):00$`).test(row.usage_date_utc ?? "")) throw new Error("Unsupported usage row or date"); + const hour = Date.parse(`${row.usage_date_utc!.replace(" ", "T")}:00Z`); + if (hour + 3_600_000 > asOf) throw new Error("Usage hour is not covered by the source timestamp"); + if (seenUsage.has(row.usage_date_utc!)) throw new Error("Duplicate usage hour"); + seenUsage.add(row.usage_date_utc!); + const five = count(row.usage_input_tokens_cache_write_5m), one = count(row.usage_input_tokens_cache_write_1h); + addTokens(providerTokens, { input: count(row.usage_input_tokens_no_cache), cachedRead: count(row.usage_input_tokens_cache_read), + cachedWrite: five + one, output: count(row.usage_output_tokens) }); + cacheWrite5m += five; cacheWrite1h += one; + } + let roundedTokenCost = 0n, roundedOtherCost = 0n, roundingLower = 0n, roundingUpper = 0n; + const keyIds = new Set(); + for (const row of costs) { + scope(row); + if (row.usage_date_utc !== config.date || !/^(?:0|[1-9][0-9]*)\.[0-9]{2}$/.test(row.cost_usd ?? "") + || !validEvalLedgerId(row.api_key_id)) throw new Error("Invalid daily cost row"); + keyIds.add(row.api_key_id); + const identity = JSON.stringify([row.cost_type, row.model, row.token_type]); + if (seenCosts.has(identity)) throw new Error("Duplicate daily cost row"); + seenCosts.add(identity); + const value = usdToUnits(row.cost_usd!); + if (row.cost_type === "token") { + dimensions(row); + if (row.model !== "Claude Sonnet 5" || row.usage_type !== "message" || !["input_no_cache", "input_cache_read", "input_cache_write_5m", "input_cache_write_1h", "output"].includes(row.token_type ?? "")) throw new Error("Unsupported token cost row"); + roundedTokenCost += value; + roundingLower += value > 5_000_000n ? value - 5_000_000n : 0n; + roundingUpper += value + 5_000_000n; + } else { + if (!["code_execution", "session_runtime", "web_search"].includes(row.cost_type ?? "") || row.model !== "--" || row.token_type !== "--") throw new Error("Unsupported non-model cost row"); + roundedOtherCost += value; + } + } + if (keyIds.size !== 1 || [...seenCosts].filter(key => JSON.parse(key)[0] === "token").length !== 5) throw new Error("Cost export lacks one key and all five token categories"); + const providerCost = estimateProviderCost({ provider: "anthropic", model: price.model, complete: true, tokens: providerTokens, + cacheWrite5m, cacheWrite1h }, price); + if (providerCost === null) throw new Error("Provider usage cannot be priced"); + const observedTokens = emptyTokens(), completeTokens = emptyTokens(), missingTokenFields = emptyTokens(); + const retained = { requests: 0, completeRequests: 0, completeTokenRequests: 0, partialTokenRequests: 0, missingTokenRequests: 0, unconfirmedModelRequests: 0, knownUsd: 0n }; + for (const request of snapshot.requests) { + if (request.phase === "infrastructure") continue; + if (!manifest.executionIds.includes(request.executionId)) throw new Error("Ledger execution is outside the manifest selection"); + const reserved = lines.find(line => line.event.type === "reserve" && line.event.requestId === request.requestId); + const finished = lines.filter(line => line.event.type === "finish" && line.event.requestId === request.requestId).at(-1); + if (!reserved || Date.parse(reserved.at) < startedAt || Date.parse(reserved.at) >= endedAt || !finished + || Date.parse(finished.at) >= endedAt || Date.parse(finished.at) > asOf) throw new Error("Ledger request is outside the completed comparison window"); + const observations = loaded.state.requests.get(request.requestId)!.observations; + if (!observations.length || observations.some(row => row.provider !== "anthropic" + || row.model !== price.model && !(row.model === undefined && row.classification === "unknown" && !row.final))) throw new Error("Ledger contains unclassified or different model requests"); + if (observations.every(row => row.model === undefined)) retained.unconfirmedModelRequests++; + if (request.conflict) throw new Error("Ledger cost observations conflict"); + retained.requests++; + if (request.receiptComplete) retained.completeRequests++; + retained.knownUsd += usdToUnits(request.costUsd ?? "0"); + const tokenEvidence = observations.filter(row => row.tokens && tokenKeys.some(key => row.tokens![key] !== undefined)); + const selected = emptyTokens(); + for (const key of tokenKeys) { + const present = tokenEvidence.filter(row => row.tokens![key] !== undefined); + const finalized = present.filter(row => row.final); + const values = new Set((finalized.length ? finalized : present).map(row => row.tokens![key]!)); + if (values.size > 1) throw new Error("Duplicate observations disagree on tokens"); + if (values.size) selected[key] = [...values][0]!; + else missingTokenFields[key]++; + } + addTokens(observedTokens, selected); + if (!tokenEvidence.length) retained.missingTokenRequests++; + else if (tokenEvidence.some(row => row.final && tokenKeys.every(key => row.tokens![key] !== undefined))) { + retained.completeTokenRequests++; + addTokens(completeTokens, selected); + } else retained.partialTokenRequests++; + } + if (!retained.requests) throw new Error("Ledger has no comparable model requests"); + const sources = [ + ["manifest", manifestBytes], ["ledger", ledgerBytes], ["usage_export", usageBytes], ["cost_export", costBytes], + ] as const; + const priced = usdToUnits(providerCost); + return { + version: 1, kind: "anthropic_provider_window_comparison", campaignId: snapshot.campaignId, + observedAt: new Date(now).toISOString(), sourceDataAsOf: config.sourceDataAsOf, + window: { startedAt: new Date(startedAt).toISOString(), endedAt: new Date(endedAt).toISOString(), currentDayPartial: asOf < endedAt }, + scopeSha256: hash(JSON.stringify([config.workspace, config.apiKeyLabel, [...keyIds][0]])), + sources: sources.map(([kind, bytes]) => ({ kind, sha256: hash(bytes) })), journalHash: loaded.previous, + model: price.model, pricingVersion: price.version, + provider: { tokens: providerTokens, cacheWrite5m, cacheWrite1h, tokenEstimateUsd: providerCost, + roundedTokenCostUsd: money(roundedTokenCost), roundedOtherCostUsd: money(roundedOtherCost), + roundedTokenCostIntervalUsd: { lowerInclusive: money(roundingLower), upperExclusive: money(roundingUpper) }, + tokenEstimateConsistentWithRoundedCost: priced >= roundingLower && priced < roundingUpper }, + retained: { ...retained, knownUsd: money(retained.knownUsd), observedTokens, completeTokens, missingTokenFields }, + difference: { tokens: Object.fromEntries(tokenKeys.map(key => [key, providerTokens[key] - observedTokens[key]])), + estimatedUsd: money(priced - retained.knownUsd) }, + invoiceReconciled: false, requestCostsAllocated: false, holdsReleased: false, + limitations: [ + "This is a scope/window comparison. Matching labels do not prove that the dedicated key had no other use.", + "Provider exports can lag and round costs. Token pricing uses the retained manifest, not an invoice or discount calculation.", + "Each known token category counts at most once per request, independently of cost. Final token evidence takes precedence; missing fields and partial receipts remain explicit.", + "Do not add the provider total to the retained subtotal or distribute their difference across requests. Journals and admission holds are unchanged.", + "Non-model export charges, infrastructure, storage and tracing remain separate from the model token comparison.", + ], + }; +} + +export async function writeAnthropicBillingComparison(configPath: string, output: string) { + const config = JSON.parse((await boundedFile(configPath)).toString("utf8")) as AnthropicBillingComparisonConfig; + const comparison = await compareAnthropicBilling(config); + await writeFile(path.resolve(output), `${JSON.stringify(comparison, null, 2)}\n`, { flag: "wx", mode: 0o600 }); + return comparison; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const [config, output, ...extra] = process.argv.slice(2); + if (!config || !output || extra.length) throw new Error("Usage: anthropic-billing-comparison.ts "); + const result = await writeAnthropicBillingComparison(config, output); + console.log(JSON.stringify({ campaignId: result.campaignId, tokenEstimateUsd: result.provider.tokenEstimateUsd, + unmatchedEstimateUsd: result.difference.estimatedUsd, invoiceReconciled: false, holdsReleased: false })); +}