Address cost provenance review feedback

This commit is contained in:
Dotta committed 2026-05-19 14:33:39 -05:00
1 parent 3b7667d5a8
commit 4fa03c1b27
5 files changed
+108 -7

No files matched your search

+3 -1
View File
@@ -401,6 +401,7 @@ async function applyRepairs(sql: PostgresSql, repairs: Repair[]) {
const affectedAgents = new Map<string, string>();
const affectedCompanies = new Set<string>();
let updatedEvents = 0;
await sql.begin(async (tx) => {
for (const repair of repairs) {
const result = await tx`
@@ -417,6 +418,7 @@ async function applyRepairs(sql: PostgresSql, repairs: Repair[]) {
AND (input_tokens + cached_input_tokens + output_tokens) > 0
`;
if (result.count > 0) {
updatedEvents += result.count;
affectedAgents.set(repair.agentId, repair.companyId);
affectedCompanies.add(repair.companyId);
}
@@ -459,7 +461,7 @@ async function applyRepairs(sql: PostgresSql, repairs: Repair[]) {
});
return {
updatedEvents: repairs.length,
updatedEvents,
recomputedAgents: affectedAgents.size,
recomputedCompanies: affectedCompanies.size,
};
@@ -28,6 +28,26 @@ describe("resolveCostProvenance", () => {
});
});
it("keeps provider-reported zero costs as reported", () => {
expect(
resolveCostProvenance({
provider: "openai",
biller: "openai",
model: "gpt-5.5",
billingType: "metered_api",
costUsd: 0,
usage: { inputTokens: 1_000_000, outputTokens: 10_000, cachedInputTokens: 0 },
costSource: "reported",
}),
).toEqual({
costUsd: 0,
costSource: "reported",
costMetadata: {
costUsd: 0,
},
});
});
it("estimates direct OpenAI metered API cost from the maintained rate table", () => {
const resolved = resolveCostProvenance({
provider: "openai",
@@ -81,6 +101,53 @@ describe("resolveCostProvenance", () => {
expect(resolved.costMetadata.rateSource).toBe("test-rate-card");
});
it("refreshes the configured model rate cache when the env changes in-process", () => {
vi.stubEnv(
"PAPERCLIP_OPENAI_MODEL_RATES_JSON",
JSON.stringify({
"gpt-cache-test": {
inputUsdPerMillion: 1,
outputUsdPerMillion: 1,
source: "first-rate-card",
},
}),
);
expect(
resolveCostProvenance({
provider: "openai",
biller: "openai",
model: "gpt-cache-test",
billingType: "metered_api",
costUsd: null,
usage: { inputTokens: 1_000_000, outputTokens: 0 },
}).costMetadata.rateSource,
).toBe("first-rate-card");
vi.stubEnv(
"PAPERCLIP_OPENAI_MODEL_RATES_JSON",
JSON.stringify({
"gpt-cache-test": {
inputUsdPerMillion: 2,
outputUsdPerMillion: 1,
source: "second-rate-card",
},
}),
);
const resolved = resolveCostProvenance({
provider: "openai",
biller: "openai",
model: "gpt-cache-test",
billingType: "metered_api",
costUsd: null,
usage: { inputTokens: 1_000_000, outputTokens: 0 },
});
expect(resolved.costUsd).toBe(2);
expect(resolved.costMetadata.rateSource).toBe("second-rate-card");
});
it("marks unpriced metered usage unavailable when no reported cost or rate exists", () => {
expect(
resolveCostProvenance({
+18 -4
View File
@@ -28,9 +28,21 @@ const DEFAULT_OPENAI_MODEL_RATES: Record<string, Rate> = {
"gpt-5.2": { inputUsdPerMillion: 1.75, cachedInputUsdPerMillion: 0.175, outputUsdPerMillion: 14, source: "openai_api_pricing_2026-05-19" },
};
let configuredRatesCacheRaw: string | undefined;
let configuredRatesCache: Record<string, Rate> | null = null;
function readConfiguredOpenAiRates(): Record<string, Rate> {
const raw = process.env.PAPERCLIP_OPENAI_MODEL_RATES_JSON;
if (!raw?.trim()) return {};
if (configuredRatesCache !== null && configuredRatesCacheRaw === raw) {
return configuredRatesCache;
}
configuredRatesCacheRaw = raw;
if (!raw?.trim()) {
configuredRatesCache = {};
return configuredRatesCache;
}
try {
const parsed = JSON.parse(raw) as Record<string, unknown>;
const rates: Record<string, Rate> = {};
@@ -50,9 +62,11 @@ function readConfiguredOpenAiRates(): Record<string, Rate> {
source: typeof record.source === "string" && record.source.trim() ? record.source.trim() : "env:PAPERCLIP_OPENAI_MODEL_RATES_JSON",
};
}
return rates;
configuredRatesCache = rates;
return configuredRatesCache;
} catch {
return {};
configuredRatesCache = {};
return configuredRatesCache;
}
}
@@ -117,7 +131,7 @@ export function resolveCostProvenance(result: Pick<
AdapterExecutionResult,
"provider" | "biller" | "model" | "billingType" | "costUsd" | "costSource" | "costMetadata" | "usage"
>): ResolvedCostProvenance {
if (typeof result.costUsd === "number" && Number.isFinite(result.costUsd) && result.costUsd > 0) {
if (typeof result.costUsd === "number" && Number.isFinite(result.costUsd) && result.costUsd >= 0) {
return {
costUsd: result.costUsd,
costSource: result.costSource === "estimated" ? "estimated" : "reported",
+6 -2
View File
@@ -30,6 +30,7 @@ export function unavailableMeteredTokens(totals: CostProvenanceTotals | null | u
export function hasCostProvenanceWarning(totals: CostProvenanceTotals | null | undefined) {
return (totals?.estimatedMeteredCostCents ?? 0) > 0 ||
(totals?.estimatedMeteredEventCount ?? 0) > 0 ||
unavailableMeteredTokens(totals) > 0 ||
(totals?.unavailableMeteredEventCount ?? 0) > 0;
}
@@ -37,9 +38,12 @@ export function hasCostProvenanceWarning(totals: CostProvenanceTotals | null | u
export function formatCostProvenanceSummary(totals: CostProvenanceTotals | null | undefined) {
const parts: string[] = [];
const estimatedCost = totals?.estimatedMeteredCostCents ?? 0;
const estimatedEvents = totals?.estimatedMeteredEventCount ?? 0;
const unpricedTokens = unavailableMeteredTokens(totals);
if (estimatedCost > 0) {
parts.push(`${formatCents(estimatedCost)} estimated metered API`);
} else if (estimatedEvents > 0) {
parts.push(`${estimatedEvents} estimated metered API event${estimatedEvents === 1 ? "" : "s"}`);
}
if (unpricedTokens > 0) {
parts.push(`${formatTokens(unpricedTokens)} unpriced metered API`);
@@ -75,9 +79,9 @@ export function CostProvenanceNotice({
return (
<div className={cn("border border-amber-300/70 bg-amber-50/60 px-3 py-2 text-xs leading-5 text-amber-900 dark:border-amber-400/40 dark:bg-amber-400/10 dark:text-amber-200", className)}>
{estimatedCost > 0 ? (
{estimatedCost > 0 || estimatedEvents > 0 ? (
<div>
{formatCents(estimatedCost)} is estimated metered API spend
{estimatedCost > 0 ? `${formatCents(estimatedCost)} is estimated metered API spend` : "Metered API spend is estimated"}
{estimatedEvents > 0 ? ` across ${estimatedEvents} event${estimatedEvents === 1 ? "" : "s"}` : ""}.
</div>
) : null}
+14
View File
@@ -4,6 +4,7 @@ import { act } from "react";
import { createRoot } from "react-dom/client";
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { formatCostProvenanceSummary, hasCostProvenanceWarning } from "../components/CostProvenanceNotice";
import { Costs } from "./Costs";
const mockCostsApi = vi.hoisted(() => ({
@@ -188,4 +189,17 @@ describe("Costs", () => {
expect(container.textContent).toContain("$2.50 estimated metered API");
expect(container.textContent).toContain("1.3k unpriced metered API");
});
it("warns for estimated metered events even when spend rounds to zero cents", () => {
const zeroCentEstimated = {
estimatedMeteredCostCents: 0,
estimatedMeteredInputTokens: 1,
estimatedMeteredCachedInputTokens: 0,
estimatedMeteredOutputTokens: 1,
estimatedMeteredEventCount: 1,
};
expect(hasCostProvenanceWarning(zeroCentEstimated)).toBe(true);
expect(formatCostProvenanceSummary(zeroCentEstimated)).toBe("1 estimated metered API event");
});
});