mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-23 09:07:46 +02:00
Normalize historical OpenAI token layouts before report aggregation
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin FoleyandPaperclip committed
1 parent
236c8adee4
commit
1b790095ed
4 files changed
+67
-14
No files matched your search
@@ -42,7 +42,7 @@ Reservations are estimates, not provider-enforced spending caps. Delayed receipt
|
||||
|
||||
Cost and finance endpoints default to the current UTC month through now. Explicit `from`/`to` bounds remain inclusive; `period=all` requests all time. Reversed or malformed bounds are rejected. CLI cost/finance reads expose `--from`, `--to`, and `--all-time`. The board sends explicit ranges, and month/year-to-date starts use UTC. Rolling spend excludes future events.
|
||||
|
||||
Cost summaries expose `eventCount`, `estimatedEventCount`, `unpricedEventCount`, `pendingRunCount`, and `pricingComplete`. Estimated charges count toward budgets, while the UI distinguishes estimates from provider-reported dollars. Agent totals and expanded model rows show **Estimated** when every charge in that group is estimated, or **Partially estimated** for a mix. Groups with no estimates have no estimate badge. These labels follow the selected date range and use ledger-event counts, not run counts; they do not infer pricing status from the provider or model name. `pricingComplete` means no missing prices or pending receipts; it does not mean invoice reconciliation has occurred. The Costs page warns when totals are incomplete. Provider cards include cached tokens and calculate subscription shares using each token once. Provider spend is compared with the real company budget, not a fabricated proportional allocation. Run displays prefer cache-adjusted prices and preserve explicit zeroes.
|
||||
Cost summaries expose `eventCount`, `estimatedEventCount`, `unpricedEventCount`, `pendingRunCount`, and `pricingComplete`. Estimated charges count toward budgets, while the UI distinguishes estimates from provider-reported dollars. Agent totals and expanded model rows show **Estimated** when every charge in that group is estimated, or **Partially estimated** for a mix. Groups with no estimates have no estimate badge. These labels follow the selected date range and use ledger-event counts, not run counts; they do not infer pricing status from the provider or model name. `pricingComplete` means no missing prices or pending receipts; it does not mean invoice reconciliation has occurred. The Costs page warns when totals are incomplete. Provider cards include cached tokens and calculate subscription shares using each token once. Reports normalize historical OpenAI events without a receipt fingerprint from inclusive input to exclusive input before grouping, so mixed historical/new totals count cache reads once across agents, models, providers, billers, projects, issues, and rolling windows. This read-time conversion preserves original ledger rows and charges; legacy Anthropic input already excludes cache reads. Provider spend is compared with the real company budget, not a fabricated proportional allocation. Run displays prefer cache-adjusted prices and preserve explicit zeroes.
|
||||
|
||||
Gemini token normalization follows the [CLI stream statistics](https://github.com/google-gemini/gemini-cli/blob/main/packages/core/src/output/stream-json-formatter.ts) and [API usage metadata](https://ai.google.dev/api/generate-content#UsageMetadata): cached prompt tokens are separated from input, and thinking tokens contribute to output.
|
||||
|
||||
|
||||
@@ -41,6 +41,49 @@ databaseDescribe("cost accounting reliability (PostgreSQL)", () => {
|
||||
return { company, agent, project, issue, goal, run, receipt };
|
||||
}
|
||||
|
||||
it("counts cache reads once across mixed historical and receipt-backed reports without changing charges", async () => {
|
||||
const f = await fixture();
|
||||
const costs = costService(db);
|
||||
const scope = { agentId: f.agent.id, issueId: f.issue.id, projectId: f.project.id, heartbeatRunId: f.run.id,
|
||||
provider: "openai", biller: "openai", model: "gpt-test", occurredAt: new Date() };
|
||||
// These two layouts both describe 100 input (80 cached) and 10 output.
|
||||
for (const billingType of ["metered_api", "subscription_included"]) {
|
||||
await db.insert(costEvents).values({ companyId: f.company.id, ...scope, billingType,
|
||||
inputTokens: 100, cachedInputTokens: 80, outputTokens: 10, costCents: 7 });
|
||||
await costs.createEvent(f.company.id, { ...scope, billingType,
|
||||
inputTokens: 20, cachedInputTokens: 80, outputTokens: 10, costCents: 11 });
|
||||
}
|
||||
const [provider, models] = await Promise.all([costs.byProvider(f.company.id), costs.byAgentModel(f.company.id)]);
|
||||
for (const rows of [provider, models]) {
|
||||
expect(rows).toHaveLength(2);
|
||||
for (const row of rows) {
|
||||
expect(row).toMatchObject({ inputTokens: 40, cachedInputTokens: 160, outputTokens: 20, costCents: 18 });
|
||||
expect(row.inputTokens + row.cachedInputTokens + row.outputTokens).toBe(220);
|
||||
}
|
||||
}
|
||||
const [agent] = await costs.byAgent(f.company.id);
|
||||
const [biller] = await costs.byBiller(f.company.id);
|
||||
const [project] = await costs.byProject(f.company.id);
|
||||
const issue = await costs.issueTreeSummary(f.company.id, f.issue.id);
|
||||
const windows = await costs.windowSpend(f.company.id);
|
||||
expect(windows).toHaveLength(3);
|
||||
for (const row of [agent, biller, project, issue, ...windows]) {
|
||||
expect(row).toMatchObject({ inputTokens: 80, cachedInputTokens: 320, outputTokens: 40, costCents: 36 });
|
||||
expect(row.inputTokens + row.cachedInputTokens + row.outputTokens).toBe(440);
|
||||
}
|
||||
for (const row of [agent, biller, provider.find(row => row.billingType === "subscription_included")!]) {
|
||||
expect(row).toMatchObject({ subscriptionInputTokens: 40, subscriptionCachedInputTokens: 160, subscriptionOutputTokens: 20 });
|
||||
}
|
||||
// Reading reports never rewrites historical evidence or retroactively bills.
|
||||
const stored = await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id));
|
||||
expect(stored.filter(row => row.receiptHash === null).map(row => row.inputTokens)).toEqual([100, 100]);
|
||||
expect((await costs.summary(f.company.id)).spendCents).toBe(36);
|
||||
// Legacy Anthropic input was already exclusive; retain its cache semantics.
|
||||
await db.insert(costEvents).values({ companyId: f.company.id, ...scope, provider: "anthropic", billingType: "metered_api",
|
||||
inputTokens: 100, cachedInputTokens: 80, outputTokens: 10, costCents: 0 });
|
||||
expect((await costs.byProvider(f.company.id)).find(row => row.provider === "anthropic")).toMatchObject({ inputTokens: 100, cachedInputTokens: 80, outputTokens: 10 });
|
||||
});
|
||||
|
||||
it.each(["issueId", "projectId", "goalId", "heartbeatRunId"] as const)("rejects a foreign-company %s without writing anything", async (field) => {
|
||||
const a = await fixture(); const b = await fixture();
|
||||
const foreignIds = { issueId: b.issue.id, projectId: b.project.id, goalId: b.goal.id, heartbeatRunId: b.run.id };
|
||||
|
||||
@@ -551,7 +551,9 @@ describeEmbeddedPostgres("cost and finance aggregate overflow handling", () => {
|
||||
const [byAgentModelRow] = await costs.byAgentModel(companyId, range);
|
||||
|
||||
expect(byAgentRow?.costCents).toBe(4_000_000_000);
|
||||
expect(byAgentRow?.inputTokens).toBe(4_000_000_000);
|
||||
// Historical OpenAI input includes these ten cached tokens.
|
||||
expect(byAgentRow?.inputTokens).toBe(3_999_999_990);
|
||||
expect(byAgentRow!.inputTokens + byAgentRow!.cachedInputTokens).toBe(4_000_000_000);
|
||||
expect(byProjectRow?.costCents).toBe(4_000_000_000);
|
||||
expect(byAgentModelRow?.costCents).toBe(4_000_000_000);
|
||||
});
|
||||
@@ -701,7 +703,7 @@ describeEmbeddedPostgres("cost and finance aggregate overflow handling", () => {
|
||||
includeDescendants: true,
|
||||
costCents: 600,
|
||||
costCentsExact: "600.0000000",
|
||||
inputTokens: 60,
|
||||
inputTokens: 54,
|
||||
cachedInputTokens: 6,
|
||||
outputTokens: 12,
|
||||
runCount: 0,
|
||||
|
||||
@@ -16,7 +16,15 @@ export type { CostDateRange } from "./cost-date-range.js";
|
||||
const METERED_BILLING_TYPE = "metered_api";
|
||||
const SUBSCRIPTION_BILLING_TYPES = ["subscription_included", "subscription_overage"] as const;
|
||||
|
||||
function sumAsNumber(column: typeof costEvents.costCents | typeof costEvents.inputTokens | typeof costEvents.cachedInputTokens | typeof costEvents.outputTokens) {
|
||||
// Pre-receipt OpenAI events stored cache reads inside input. Normalize each
|
||||
// historical row before aggregation so mixed old/new groups share the current
|
||||
// exclusive-input contract. Preserve the original ledger and monetary amounts.
|
||||
const ordinaryInputTokens = sql<number>`case
|
||||
when ${costEvents.receiptHash} is null and ${costEvents.provider} = 'openai'
|
||||
then greatest(0, ${costEvents.inputTokens} - ${costEvents.cachedInputTokens})
|
||||
else ${costEvents.inputTokens} end`;
|
||||
|
||||
function sumAsNumber(column: typeof costEvents.costCents | typeof costEvents.inputTokens | typeof costEvents.cachedInputTokens | typeof costEvents.outputTokens | ReturnType<typeof sql>) {
|
||||
return sql<number>`coalesce(sum(${column}), 0)::double precision`;
|
||||
}
|
||||
|
||||
@@ -305,7 +313,7 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
|
||||
issueCount: sql<number>`count(distinct ${issues.id})::int`,
|
||||
costCents: sumAsNumber(costEvents.costCents),
|
||||
costCentsExact: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
|
||||
inputTokens: sumAsNumber(costEvents.inputTokens),
|
||||
inputTokens: sumAsNumber(ordinaryInputTokens),
|
||||
cachedInputTokens: sumAsNumber(costEvents.cachedInputTokens),
|
||||
outputTokens: sumAsNumber(costEvents.outputTokens),
|
||||
})
|
||||
@@ -362,7 +370,7 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
|
||||
estimatedEventCount: sql<number>`count(*) filter (where ${costEvents.costStatus} = 'estimated')::int`,
|
||||
costCents: sumAsNumber(costEvents.costCents),
|
||||
costCentsExact: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
|
||||
inputTokens: sumAsNumber(costEvents.inputTokens),
|
||||
inputTokens: sumAsNumber(ordinaryInputTokens),
|
||||
cachedInputTokens: sumAsNumber(costEvents.cachedInputTokens),
|
||||
outputTokens: sumAsNumber(costEvents.outputTokens),
|
||||
apiRunCount:
|
||||
@@ -372,7 +380,7 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
|
||||
subscriptionCachedInputTokens:
|
||||
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${costEvents.cachedInputTokens} else 0 end), 0)::double precision`,
|
||||
subscriptionInputTokens:
|
||||
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${costEvents.inputTokens} else 0 end), 0)::double precision`,
|
||||
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${ordinaryInputTokens} else 0 end), 0)::double precision`,
|
||||
subscriptionOutputTokens:
|
||||
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${costEvents.outputTokens} else 0 end), 0)::double precision`,
|
||||
})
|
||||
@@ -401,7 +409,7 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
|
||||
model: costEvents.model,
|
||||
costCents: sumAsNumber(costEvents.costCents),
|
||||
costCentsExact: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
|
||||
inputTokens: sumAsNumber(costEvents.inputTokens),
|
||||
inputTokens: sumAsNumber(ordinaryInputTokens),
|
||||
cachedInputTokens: sumAsNumber(costEvents.cachedInputTokens),
|
||||
outputTokens: sumAsNumber(costEvents.outputTokens),
|
||||
apiRunCount:
|
||||
@@ -411,7 +419,7 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
|
||||
subscriptionCachedInputTokens:
|
||||
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${costEvents.cachedInputTokens} else 0 end), 0)::double precision`,
|
||||
subscriptionInputTokens:
|
||||
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${costEvents.inputTokens} else 0 end), 0)::double precision`,
|
||||
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${ordinaryInputTokens} else 0 end), 0)::double precision`,
|
||||
subscriptionOutputTokens:
|
||||
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${costEvents.outputTokens} else 0 end), 0)::double precision`,
|
||||
})
|
||||
@@ -432,7 +440,7 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
|
||||
biller: costEvents.biller,
|
||||
costCents: sumAsNumber(costEvents.costCents),
|
||||
costCentsExact: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
|
||||
inputTokens: sumAsNumber(costEvents.inputTokens),
|
||||
inputTokens: sumAsNumber(ordinaryInputTokens),
|
||||
cachedInputTokens: sumAsNumber(costEvents.cachedInputTokens),
|
||||
outputTokens: sumAsNumber(costEvents.outputTokens),
|
||||
apiRunCount:
|
||||
@@ -442,7 +450,7 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
|
||||
subscriptionCachedInputTokens:
|
||||
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${costEvents.cachedInputTokens} else 0 end), 0)::double precision`,
|
||||
subscriptionInputTokens:
|
||||
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${costEvents.inputTokens} else 0 end), 0)::double precision`,
|
||||
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${ordinaryInputTokens} else 0 end), 0)::double precision`,
|
||||
subscriptionOutputTokens:
|
||||
sql<number>`coalesce(sum(case when ${costEvents.billingType} in (${sql.join(SUBSCRIPTION_BILLING_TYPES.map((value) => sql`${value}`), sql`, `)}) then ${costEvents.outputTokens} else 0 end), 0)::double precision`,
|
||||
providerCount: sql<number>`count(distinct ${costEvents.provider})::int`,
|
||||
@@ -476,7 +484,7 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
|
||||
biller: sql<string>`case when count(distinct ${costEvents.biller}) = 1 then min(${costEvents.biller}) else 'mixed' end`,
|
||||
costCents: sumAsNumber(costEvents.costCents),
|
||||
costCentsExact: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
|
||||
inputTokens: sumAsNumber(costEvents.inputTokens),
|
||||
inputTokens: sumAsNumber(ordinaryInputTokens),
|
||||
cachedInputTokens: sumAsNumber(costEvents.cachedInputTokens),
|
||||
outputTokens: sumAsNumber(costEvents.outputTokens),
|
||||
})
|
||||
@@ -531,7 +539,7 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
|
||||
estimatedEventCount: sql<number>`count(*) filter (where ${costEvents.costStatus} = 'estimated')::int`,
|
||||
costCents: sumAsNumber(costEvents.costCents),
|
||||
costCentsExact: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
|
||||
inputTokens: sumAsNumber(costEvents.inputTokens),
|
||||
inputTokens: sumAsNumber(ordinaryInputTokens),
|
||||
cachedInputTokens: sumAsNumber(costEvents.cachedInputTokens),
|
||||
outputTokens: sumAsNumber(costEvents.outputTokens),
|
||||
})
|
||||
@@ -595,7 +603,7 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
|
||||
projectName: projects.name,
|
||||
costCents: costCentsExpr,
|
||||
costCentsExact: sql<string>`coalesce(sum(${costEvents.costCents}), 0)::text`,
|
||||
inputTokens: sumAsNumber(costEvents.inputTokens),
|
||||
inputTokens: sumAsNumber(ordinaryInputTokens),
|
||||
cachedInputTokens: sumAsNumber(costEvents.cachedInputTokens),
|
||||
outputTokens: sumAsNumber(costEvents.outputTokens),
|
||||
})
|
||||
|
||||
Reference in new issue
Block a user