diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index 2753512bd51..b4e27098089 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -278,13 +278,19 @@ Invariants: - `goal_id` uuid fk `goals.id` null - `billing_code` text null - `provider` text not null +- `biller` text not null +- `billing_type` text not null - `model` text not null - `input_tokens` int not null default 0 +- `cached_input_tokens` int not null default 0 - `output_tokens` int not null default 0 - `cost_cents` int not null +- `cost_source` enum/text: `reported | estimated | unavailable` not null default `unavailable` +- `cost_metadata` jsonb null - `occurred_at` timestamptz not null Invariant: each event must attach to agent and company; rollups are aggregation, never manually edited. +When model spend is estimated from a maintained pricing table instead of directly reported by a provider, the row must be marked `estimated` and include enough metadata to explain the rate source and token inputs. Unpriced token usage should still write a visible `unavailable` provenance row instead of silently disappearing. ## 7.10 `approvals` diff --git a/packages/adapter-utils/src/types.ts b/packages/adapter-utils/src/types.ts index 3d87f7a284c..d39282cb0c2 100644 --- a/packages/adapter-utils/src/types.ts +++ b/packages/adapter-utils/src/types.ts @@ -31,6 +31,7 @@ export interface UsageSummary { inputTokens: number; outputTokens: number; cachedInputTokens?: number; + reasoningOutputTokens?: number; } export type AdapterBillingType = @@ -87,6 +88,8 @@ export interface AdapterExecutionResult { model?: string | null; billingType?: AdapterBillingType | null; costUsd?: number | null; + costSource?: "reported" | "estimated" | "unavailable" | null; + costMetadata?: Record | null; resultJson?: Record | null; runtimeServices?: AdapterRuntimeServiceReport[]; summary?: string | null; diff --git a/packages/adapters/codex-local/src/server/execute.ts b/packages/adapters/codex-local/src/server/execute.ts index 45b856b79a3..865f39955c4 100644 --- a/packages/adapters/codex-local/src/server/execute.ts +++ b/packages/adapters/codex-local/src/server/execute.ts @@ -813,10 +813,18 @@ export async function execute(ctx: AdapterExecutionContext): Promise { inputTokens: 10, cachedInputTokens: 2, outputTokens: 4, + reasoningOutputTokens: 0, }, + costUsd: null, + costField: null, errorMessage: "resume failed", }); }); @@ -61,10 +64,49 @@ describe("parseCodexJsonl", () => { inputTokens: 10, cachedInputTokens: 2, outputTokens: 4, + reasoningOutputTokens: 0, }, + costUsd: null, + costField: null, errorMessage: null, }); }); + + it("captures reported cost aliases and reasoning tokens from turn usage", () => { + const stdout = [ + JSON.stringify({ + type: "turn.completed", + usage: { + input_tokens: 1_000, + cached_input_tokens: 200, + output_tokens: 300, + reasoning_output_tokens: 700, + cost: { total: "0.0123" }, + }, + }), + JSON.stringify({ + type: "turn.completed", + usage: { + input_tokens: 2_000, + cached_input_tokens: 500, + output_tokens: 400, + output_tokens_details: { reasoning_tokens: 800 }, + }, + total_cost_usd: 0.0456, + }), + ].join("\n"); + + expect(parseCodexJsonl(stdout)).toMatchObject({ + usage: { + inputTokens: 2_000, + cachedInputTokens: 500, + outputTokens: 400, + reasoningOutputTokens: 800, + }, + costUsd: 0.0456, + costField: "total_cost_usd", + }); + }); }); describe("isCodexUnknownSessionError", () => { diff --git a/packages/adapters/codex-local/src/server/parse.ts b/packages/adapters/codex-local/src/server/parse.ts index 679a3f8f4df..28e1eeb61c9 100644 --- a/packages/adapters/codex-local/src/server/parse.ts +++ b/packages/adapters/codex-local/src/server/parse.ts @@ -15,10 +15,13 @@ export function parseCodexJsonl(stdout: string) { let sessionId: string | null = null; let finalMessage: string | null = null; let errorMessage: string | null = null; + let costUsd: number | null = null; + let costField: string | null = null; const usage = { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0, + reasoningOutputTokens: 0, }; for (const rawLine of stdout.split(/\r?\n/)) { @@ -29,6 +32,12 @@ export function parseCodexJsonl(stdout: string) { if (!event) continue; const type = asString(event.type, ""); + const eventCost = readCodexCostUsd(event); + if (eventCost) { + costUsd = eventCost.costUsd; + costField = eventCost.field; + } + if (type === "thread.started") { sessionId = asString(event.thread_id, sessionId ?? "") || sessionId; continue; @@ -54,10 +63,24 @@ export function parseCodexJsonl(stdout: string) { usage.inputTokens = asNumber(usageObj.input_tokens, usage.inputTokens); usage.cachedInputTokens = asNumber(usageObj.cached_input_tokens, usage.cachedInputTokens); usage.outputTokens = asNumber(usageObj.output_tokens, usage.outputTokens); + usage.reasoningOutputTokens = asNumber( + usageObj.reasoning_output_tokens, + asNumber(parseObject(usageObj.output_tokens_details).reasoning_tokens, usage.reasoningOutputTokens), + ); continue; } if (type === "turn.failed") { + const usageObj = parseObject(event.usage); + if (Object.keys(usageObj).length > 0) { + usage.inputTokens = asNumber(usageObj.input_tokens, usage.inputTokens); + usage.cachedInputTokens = asNumber(usageObj.cached_input_tokens, usage.cachedInputTokens); + usage.outputTokens = asNumber(usageObj.output_tokens, usage.outputTokens); + usage.reasoningOutputTokens = asNumber( + usageObj.reasoning_output_tokens, + asNumber(parseObject(usageObj.output_tokens_details).reasoning_tokens, usage.reasoningOutputTokens), + ); + } const err = parseObject(event.error); const msg = asString(err.message, "").trim(); if (msg) errorMessage = msg; @@ -68,10 +91,39 @@ export function parseCodexJsonl(stdout: string) { sessionId, summary: finalMessage?.trim() ?? "", usage, + costUsd, + costField, errorMessage, }; } +function positiveNumber(value: unknown): number | null { + const parsed = typeof value === "string" ? Number(value) : asNumber(value, Number.NaN); + return Number.isFinite(parsed) && parsed > 0 ? parsed : null; +} + +function readCodexCostUsd(event: Record): { costUsd: number; field: string } | null { + const usage = parseObject(event.usage); + const usageCost = positiveNumber(usage.cost); + if (usageCost != null) return { costUsd: usageCost, field: "usage.cost" }; + const usageCostRaw = usage.cost; + const usageCostObj = parseObject(usageCostRaw); + for (const key of ["total", "total_usd", "usd", "costUsd", "cost_usd", "total_cost_usd"] as const) { + const value = positiveNumber(usageCostObj[key]); + if (value != null) return { costUsd: value, field: `usage.cost.${key}` }; + } + + for (const key of ["costUsd", "cost_usd", "total_cost_usd"] as const) { + const value = positiveNumber(event[key]); + if (value != null) return { costUsd: value, field: key }; + } + + const usageCostUsd = positiveNumber(usage.costUsd ?? usage.cost_usd ?? usage.total_cost_usd); + if (usageCostUsd != null) return { costUsd: usageCostUsd, field: "usage.costUsd" }; + + return null; +} + export function isCodexUnknownSessionError(stdout: string, stderr: string): boolean { const haystack = `${stdout}\n${stderr}` .split(/\r?\n/) diff --git a/packages/db/src/migrations/0087_cost_event_provenance.sql b/packages/db/src/migrations/0087_cost_event_provenance.sql new file mode 100644 index 00000000000..39c0e7aea89 --- /dev/null +++ b/packages/db/src/migrations/0087_cost_event_provenance.sql @@ -0,0 +1,9 @@ +ALTER TABLE "cost_events" ADD COLUMN IF NOT EXISTS "cost_source" text NOT NULL DEFAULT 'unavailable';--> statement-breakpoint +ALTER TABLE "cost_events" ADD COLUMN IF NOT EXISTS "cost_metadata" jsonb;--> statement-breakpoint +UPDATE "cost_events" +SET "cost_source" = CASE + WHEN "cost_cents" > 0 THEN 'reported' + WHEN "billing_type" = 'metered_api' AND ("input_tokens" > 0 OR "cached_input_tokens" > 0 OR "output_tokens" > 0) THEN 'unavailable' + ELSE 'unavailable' +END +WHERE "cost_source" = 'unavailable'; diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index a47bae7c2d1..166c93f7e97 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -610,6 +610,13 @@ "when": 1778976000000, "tag": "0086_routine_env_runtime_contract", "breakpoints": true + }, + { + "idx": 87, + "version": "7", + "when": 1779210381000, + "tag": "0087_cost_event_provenance", + "breakpoints": true } ] } diff --git a/packages/db/src/schema/cost_events.ts b/packages/db/src/schema/cost_events.ts index f8a36847e69..4e7694e48fd 100644 --- a/packages/db/src/schema/cost_events.ts +++ b/packages/db/src/schema/cost_events.ts @@ -1,4 +1,4 @@ -import { pgTable, uuid, text, timestamp, integer, index } from "drizzle-orm/pg-core"; +import { pgTable, uuid, text, timestamp, integer, index, jsonb } from "drizzle-orm/pg-core"; import { companies } from "./companies.js"; import { agents } from "./agents.js"; import { issues } from "./issues.js"; @@ -25,6 +25,8 @@ export const costEvents = pgTable( cachedInputTokens: integer("cached_input_tokens").notNull().default(0), outputTokens: integer("output_tokens").notNull().default(0), costCents: integer("cost_cents").notNull(), + costSource: text("cost_source").notNull().default("unavailable"), + costMetadata: jsonb("cost_metadata").$type | null>(), occurredAt: timestamp("occurred_at", { withTimezone: true }).notNull(), createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), }, diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index dcd92bcfe62..510befc0941 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -468,6 +468,7 @@ export type { BudgetOverview, BudgetPolicyUpsertInput, BudgetIncidentResolutionInput, + CostSource, CostEvent, CostSummary, IssueCostSummary, @@ -922,6 +923,7 @@ export { type UpdateRoutineTrigger, type RunRoutine, type RotateRoutineTriggerSecret, + COST_SOURCES, createCostEventSchema, createFinanceEventSchema, updateBudgetSchema, diff --git a/packages/shared/src/types/cost.ts b/packages/shared/src/types/cost.ts index 11d9a7652cc..85162d110a6 100644 --- a/packages/shared/src/types/cost.ts +++ b/packages/shared/src/types/cost.ts @@ -1,5 +1,7 @@ import type { BillingType } from "../constants.js"; +export type CostSource = "reported" | "estimated" | "unavailable"; + export interface CostEvent { id: string; companyId: string; @@ -17,6 +19,8 @@ export interface CostEvent { cachedInputTokens: number; outputTokens: number; costCents: number; + costSource: CostSource; + costMetadata: Record | null; occurredAt: Date; createdAt: Date; } diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index a23547572a3..4572faab480 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -296,7 +296,7 @@ export type { RoutineExecutionIssueOrigin, RoutineListItem, } from "./routine.js"; -export type { CostEvent, CostSummary, IssueCostSummary, CostByAgent, CostByProviderModel, CostByBiller, CostByAgentModel, CostWindowSpendRow, CostByProject } from "./cost.js"; +export type { CostSource, CostEvent, CostSummary, IssueCostSummary, CostByAgent, CostByProviderModel, CostByBiller, CostByAgentModel, CostWindowSpendRow, CostByProject } from "./cost.js"; export type { FinanceEvent, FinanceSummary, FinanceByBiller, FinanceByKind } from "./finance.js"; export type { AgentWakeupResponse, diff --git a/packages/shared/src/validators/cost.ts b/packages/shared/src/validators/cost.ts index ed22d7ebc23..65cd4c32ea7 100644 --- a/packages/shared/src/validators/cost.ts +++ b/packages/shared/src/validators/cost.ts @@ -1,6 +1,8 @@ import { z } from "zod"; import { BILLING_TYPES } from "../constants.js"; +export const COST_SOURCES = ["reported", "estimated", "unavailable"] as const; + export const createCostEventSchema = z.object({ agentId: z.string().uuid(), issueId: z.string().uuid().optional().nullable(), @@ -16,6 +18,8 @@ export const createCostEventSchema = z.object({ cachedInputTokens: z.number().int().nonnegative().optional().default(0), outputTokens: z.number().int().nonnegative().optional().default(0), costCents: z.number().int().nonnegative(), + costSource: z.enum(COST_SOURCES).optional().default("unavailable"), + costMetadata: z.record(z.string(), z.unknown()).optional().nullable(), occurredAt: z.string().datetime(), }).transform((value) => ({ ...value, diff --git a/packages/shared/src/validators/index.ts b/packages/shared/src/validators/index.ts index 3380c5534a3..da17bf3642e 100644 --- a/packages/shared/src/validators/index.ts +++ b/packages/shared/src/validators/index.ts @@ -337,6 +337,7 @@ export { } from "./routine.js"; export { + COST_SOURCES, createCostEventSchema, updateBudgetSchema, type CreateCostEvent, diff --git a/server/src/__tests__/codex-local-adapter.test.ts b/server/src/__tests__/codex-local-adapter.test.ts index 84c806cd7a0..da77cb448fe 100644 --- a/server/src/__tests__/codex-local-adapter.test.ts +++ b/server/src/__tests__/codex-local-adapter.test.ts @@ -19,6 +19,7 @@ describe("codex_local parser", () => { inputTokens: 10, cachedInputTokens: 2, outputTokens: 4, + reasoningOutputTokens: 0, }); expect(parsed.errorMessage).toBe("model access denied"); }); diff --git a/server/src/__tests__/heartbeat-cost-provenance.test.ts b/server/src/__tests__/heartbeat-cost-provenance.test.ts new file mode 100644 index 00000000000..1ea9cb4414a --- /dev/null +++ b/server/src/__tests__/heartbeat-cost-provenance.test.ts @@ -0,0 +1,211 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { eq } from "drizzle-orm"; +import { + agents, + agentRuntimeState, + companies, + costEvents, + createDb, + heartbeatRuns, +} from "@paperclipai/db"; +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "./helpers/embedded-postgres.js"; + +const mockTelemetryClient = vi.hoisted(() => ({ track: vi.fn() })); +const mockTrackAgentFirstHeartbeat = vi.hoisted(() => vi.fn()); +const mockAdapterExecute = vi.hoisted(() => vi.fn()); + +vi.mock("../telemetry.ts", () => ({ + getTelemetryClient: () => mockTelemetryClient, +})); + +vi.mock("@paperclipai/shared/telemetry", async () => { + const actual = await vi.importActual( + "@paperclipai/shared/telemetry", + ); + return { + ...actual, + trackAgentFirstHeartbeat: mockTrackAgentFirstHeartbeat, + }; +}); + +vi.mock("../adapters/index.ts", async () => { + const actual = await vi.importActual("../adapters/index.ts"); + return { + ...actual, + getServerAdapter: vi.fn(() => ({ + supportsLocalAgentJwt: false, + execute: mockAdapterExecute, + })), + }; +}); + +import { heartbeatService } from "../services/heartbeat.ts"; + +const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; + +if (!embeddedPostgresSupport.supported) { + console.warn( + `Skipping embedded Postgres heartbeat cost provenance tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, + ); +} + +describeEmbeddedPostgres("heartbeat cost provenance", () => { + let db!: ReturnType; + let tempDb: Awaited> | null = null; + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("heartbeat-cost-provenance-"); + db = createDb(tempDb.connectionString); + }, 20_000); + + afterEach(async () => { + mockAdapterExecute.mockReset(); + }); + + afterAll(async () => { + await tempDb?.cleanup(); + }); + + async function seedAgent() { + const companyId = randomUUID(); + const agentId = randomUUID(); + const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`; + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix, + requireBoardApprovalForNewAgents: false, + }); + + await db.insert(agents).values({ + id: agentId, + companyId, + name: "CodexCoder", + role: "engineer", + status: "idle", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }); + + return { companyId, agentId }; + } + + async function runHeartbeatAndReadCostEvent(agentId: string) { + const heartbeat = heartbeatService(db); + const run = await heartbeat.invoke(agentId, "on_demand", {}, "manual", { + actorType: "system", + actorId: "test", + }); + expect(run).toBeTruthy(); + + let event: typeof costEvents.$inferSelect | null = null; + for (let attempt = 0; attempt < 80; attempt += 1) { + const rows = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, run!.id)); + event = rows[0] ?? event; + const [finishedRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run!.id)); + if (event && finishedRun?.status === "succeeded") return event; + if (finishedRun?.status === "failed" || finishedRun?.status === "timed_out" || finishedRun?.status === "cancelled") { + throw new Error(`Heartbeat run ended before writing a cost event: ${finishedRun.status}`); + } + await new Promise((resolve) => setTimeout(resolve, 25)); + } + + throw new Error("Timed out waiting for cost event"); + } + + it("stores provider-reported Codex cost provenance", async () => { + const { agentId } = await seedAgent(); + mockAdapterExecute.mockResolvedValueOnce({ + exitCode: 0, + signal: null, + timedOut: false, + errorMessage: null, + provider: "openai", + biller: "openai", + model: "gpt-5.5", + billingType: "metered_api", + usage: { inputTokens: 1_000, cachedInputTokens: 200, outputTokens: 300 }, + costUsd: 0.42, + costSource: "reported", + costMetadata: { providerCostField: "total_cost_usd", source: "codex_jsonl" }, + }); + + const event = await runHeartbeatAndReadCostEvent(agentId); + + expect(event.costCents).toBe(42); + expect(event.costSource).toBe("reported"); + expect(event.costMetadata).toMatchObject({ + providerCostField: "total_cost_usd", + source: "codex_jsonl", + costUsd: 0.42, + }); + }); + + it("estimates direct OpenAI metered API cost when no provider cost is reported", async () => { + const { agentId } = await seedAgent(); + mockAdapterExecute.mockResolvedValueOnce({ + exitCode: 0, + signal: null, + timedOut: false, + errorMessage: null, + provider: "openai", + biller: "openai", + model: "gpt-5.5", + billingType: "metered_api", + usage: { + inputTokens: 1_000_000, + cachedInputTokens: 100_000, + outputTokens: 10_000, + reasoningOutputTokens: 5_000, + }, + costUsd: null, + }); + + const event = await runHeartbeatAndReadCostEvent(agentId); + + expect(event.costCents).toBe(500); + expect(event.costSource).toBe("estimated"); + expect(event.costMetadata).toMatchObject({ + estimator: "openai_model_rate_table", + inputUsdPerMillion: 5, + cachedInputUsdPerMillion: 0.5, + outputUsdPerMillion: 30, + reasoningOutputTokens: 5_000, + }); + + const [state] = await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, agentId)); + expect(state?.totalCostCents).toBe(500); + }); + + it("keeps unpriced token usage visible as unavailable provenance", async () => { + const { agentId } = await seedAgent(); + mockAdapterExecute.mockResolvedValueOnce({ + exitCode: 0, + signal: null, + timedOut: false, + errorMessage: null, + provider: "openai", + biller: "openai", + model: "gpt-unknown-future", + billingType: "metered_api", + usage: { inputTokens: 10_000, cachedInputTokens: 0, outputTokens: 1_000 }, + costUsd: null, + }); + + const event = await runHeartbeatAndReadCostEvent(agentId); + + expect(event.costCents).toBe(0); + expect(event.costSource).toBe("unavailable"); + expect(event.costMetadata).toMatchObject({ + reason: "no_reported_cost_or_configured_rate", + }); + }); +}); diff --git a/server/src/__tests__/openai-cost-estimates.test.ts b/server/src/__tests__/openai-cost-estimates.test.ts new file mode 100644 index 00000000000..f8e0f7a6e55 --- /dev/null +++ b/server/src/__tests__/openai-cost-estimates.test.ts @@ -0,0 +1,100 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { resolveCostProvenance } from "../services/openai-cost-estimates.ts"; + +describe("resolveCostProvenance", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("keeps provider-reported costs as reported", () => { + expect( + resolveCostProvenance({ + provider: "openai", + biller: "openai", + model: "gpt-5.5", + billingType: "metered_api", + costUsd: 1.23, + usage: { inputTokens: 1, outputTokens: 1, cachedInputTokens: 0 }, + costSource: "reported", + costMetadata: { providerCostField: "total_cost_usd" }, + }), + ).toEqual({ + costUsd: 1.23, + costSource: "reported", + costMetadata: { + providerCostField: "total_cost_usd", + costUsd: 1.23, + }, + }); + }); + + it("estimates direct OpenAI metered API cost from the maintained rate table", () => { + const resolved = resolveCostProvenance({ + provider: "openai", + biller: "openai", + model: "gpt-5.5", + billingType: "metered_api", + costUsd: null, + usage: { + inputTokens: 1_000_000, + cachedInputTokens: 100_000, + outputTokens: 10_000, + reasoningOutputTokens: 5_000, + }, + }); + + expect(resolved.costSource).toBe("estimated"); + expect(resolved.costUsd).toBeCloseTo(5, 6); + expect(resolved.costMetadata).toMatchObject({ + estimator: "openai_model_rate_table", + inputUsdPerMillion: 5, + cachedInputUsdPerMillion: 0.5, + outputUsdPerMillion: 30, + reasoningOutputTokens: 5_000, + }); + }); + + it("uses configured model rates for unknown OpenAI models", () => { + vi.stubEnv( + "PAPERCLIP_OPENAI_MODEL_RATES_JSON", + JSON.stringify({ + "gpt-test-codex": { + inputUsdPerMillion: 2, + cachedInputUsdPerMillion: 0.2, + outputUsdPerMillion: 10, + source: "test-rate-card", + }, + }), + ); + + const resolved = resolveCostProvenance({ + provider: "openai", + biller: "openai", + model: "gpt-test-codex", + billingType: "metered_api", + costUsd: null, + usage: { inputTokens: 1_000_000, cachedInputTokens: 500_000, outputTokens: 100_000 }, + }); + + expect(resolved.costSource).toBe("estimated"); + expect(resolved.costUsd).toBeCloseTo(2.1, 6); + expect(resolved.costMetadata.rateSource).toBe("test-rate-card"); + }); + + it("marks unpriced metered usage unavailable when no reported cost or rate exists", () => { + expect( + resolveCostProvenance({ + provider: "openai", + biller: "openai", + model: "gpt-unknown-future", + billingType: "metered_api", + costUsd: null, + usage: { inputTokens: 10_000, outputTokens: 1_000 }, + }), + ).toMatchObject({ + costUsd: null, + costSource: "unavailable", + costMetadata: { reason: "no_reported_cost_or_configured_rate" }, + }); + }); +}); diff --git a/server/src/services/costs.ts b/server/src/services/costs.ts index 05008d8222d..ef2637237a3 100644 --- a/server/src/services/costs.ts +++ b/server/src/services/costs.ts @@ -71,6 +71,8 @@ export function costService(db: Db, budgetHooks: BudgetServiceHooks = {}) { biller: data.biller ?? data.provider, billingType: data.billingType ?? "unknown", cachedInputTokens: data.cachedInputTokens ?? 0, + costSource: data.costSource ?? "unavailable", + costMetadata: data.costMetadata ?? null, }) .returning() .then((rows) => rows[0]); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 8c34e99233a..3e5522bf21e 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -61,6 +61,7 @@ import type { import { createLocalAgentJwt } from "../agent-auth-jwt.js"; import { parseObject, asBoolean, asNumber, appendWithByteCap, MAX_EXCERPT_BYTES } from "../adapters/utils.js"; import { costService } from "./costs.js"; +import { resolveCostProvenance } from "./openai-cost-estimates.js"; import { trackAgentFirstHeartbeat } from "@paperclipai/shared/telemetry"; import { getTelemetryClient } from "../telemetry.js"; import { companySkillService } from "./company-skills.js"; @@ -1031,6 +1032,7 @@ type UsageTotals = { inputTokens: number; cachedInputTokens: number; outputTokens: number; + reasoningOutputTokens: number; }; type SessionCompactionDecision = { @@ -1443,6 +1445,7 @@ function normalizeUsageTotals(usage: UsageSummary | null | undefined): UsageTota inputTokens: Math.max(0, Math.floor(asNumber(usage.inputTokens, 0))), cachedInputTokens: Math.max(0, Math.floor(asNumber(usage.cachedInputTokens, 0))), outputTokens: Math.max(0, Math.floor(asNumber(usage.outputTokens, 0))), + reasoningOutputTokens: Math.max(0, Math.floor(asNumber(usage.reasoningOutputTokens, 0))), }; } @@ -1462,8 +1465,12 @@ function readRawUsageTotals(usageJson: unknown): UsageTotals | null { 0, Math.floor(asNumber(parsed.rawOutputTokens, asNumber(parsed.outputTokens, 0))), ); + const reasoningOutputTokens = Math.max( + 0, + Math.floor(asNumber(parsed.rawReasoningOutputTokens, asNumber(parsed.reasoningOutputTokens, 0))), + ); - if (inputTokens <= 0 && cachedInputTokens <= 0 && outputTokens <= 0) { + if (inputTokens <= 0 && cachedInputTokens <= 0 && outputTokens <= 0 && reasoningOutputTokens <= 0) { return null; } @@ -1471,6 +1478,7 @@ function readRawUsageTotals(usageJson: unknown): UsageTotals | null { inputTokens, cachedInputTokens, outputTokens, + reasoningOutputTokens, }; } @@ -1487,11 +1495,15 @@ function deriveNormalizedUsageDelta(current: UsageTotals | null, previous: Usage const outputTokens = current.outputTokens >= previous.outputTokens ? current.outputTokens - previous.outputTokens : current.outputTokens; + const reasoningOutputTokens = current.reasoningOutputTokens >= previous.reasoningOutputTokens + ? current.reasoningOutputTokens - previous.reasoningOutputTokens + : current.reasoningOutputTokens; return { inputTokens: Math.max(0, inputTokens), cachedInputTokens: Math.max(0, cachedInputTokens), outputTokens: Math.max(0, outputTokens), + reasoningOutputTokens: Math.max(0, reasoningOutputTokens), }; } @@ -6761,11 +6773,26 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const inputTokens = usage?.inputTokens ?? 0; const outputTokens = usage?.outputTokens ?? 0; const cachedInputTokens = usage?.cachedInputTokens ?? 0; + const reasoningOutputTokens = usage?.reasoningOutputTokens ?? 0; const billingType = normalizeLedgerBillingType(result.billingType); - const additionalCostCents = normalizeBilledCostCents(result.costUsd, billingType); - const hasTokenUsage = inputTokens > 0 || outputTokens > 0 || cachedInputTokens > 0; const provider = result.provider ?? "unknown"; const biller = resolveLedgerBiller(result); + const costProvenance = resolveCostProvenance({ + ...result, + provider, + biller, + billingType, + usage: usage + ? { + inputTokens, + cachedInputTokens, + outputTokens, + reasoningOutputTokens, + } + : result.usage, + }); + const additionalCostCents = normalizeBilledCostCents(costProvenance.costUsd, billingType); + const hasTokenUsage = inputTokens > 0 || outputTokens > 0 || cachedInputTokens > 0 || reasoningOutputTokens > 0; const ledgerScope = await resolveLedgerScopeForRun(db, agent.companyId, run); await db @@ -6799,6 +6826,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) cachedInputTokens, outputTokens, costCents: additionalCostCents, + costSource: costProvenance.costSource, + costMetadata: { + ...costProvenance.costMetadata, + ...(reasoningOutputTokens > 0 ? { reasoningOutputTokens } : {}), + }, occurredAt: new Date(), }); } @@ -7926,6 +7958,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) rawInputTokens: rawUsage.inputTokens, rawCachedInputTokens: rawUsage.cachedInputTokens, rawOutputTokens: rawUsage.outputTokens, + rawReasoningOutputTokens: rawUsage.reasoningOutputTokens, } : {}), ...(sessionUsageResolution.derivedFromSessionTotals ? { usageSource: "session_delta" } : {}), ...((nextSessionState.displayId ?? nextSessionState.legacySessionId) @@ -7940,6 +7973,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) biller: resolveLedgerBiller(adapterResult), model: readNonEmptyString(adapterResult.model) ?? "unknown", ...(adapterResult.costUsd != null ? { costUsd: adapterResult.costUsd } : {}), + ...(adapterResult.costSource ? { costSource: adapterResult.costSource } : {}), + ...(adapterResult.costMetadata ? { costMetadata: adapterResult.costMetadata } : {}), billingType: normalizeLedgerBillingType(adapterResult.billingType), } as Record) : null; diff --git a/server/src/services/openai-cost-estimates.ts b/server/src/services/openai-cost-estimates.ts new file mode 100644 index 00000000000..9a072577645 --- /dev/null +++ b/server/src/services/openai-cost-estimates.ts @@ -0,0 +1,162 @@ +import type { AdapterBillingType, AdapterExecutionResult, UsageSummary } from "@paperclipai/adapter-utils"; +import type { BillingType } from "@paperclipai/shared"; + +type CostSource = "reported" | "estimated" | "unavailable"; + +type Rate = { + inputUsdPerMillion: number; + cachedInputUsdPerMillion: number | null; + outputUsdPerMillion: number; + source: string; +}; + +export type ResolvedCostProvenance = { + costUsd: number | null; + costSource: CostSource; + costMetadata: Record; +}; + +const DEFAULT_OPENAI_MODEL_RATES: Record = { + "gpt-5.5": { inputUsdPerMillion: 5, cachedInputUsdPerMillion: 0.5, outputUsdPerMillion: 30, source: "openai_api_pricing_2026-05-19" }, + "gpt-5.5-pro": { inputUsdPerMillion: 30, cachedInputUsdPerMillion: null, outputUsdPerMillion: 180, source: "openai_api_pricing_2026-05-19" }, + "gpt-5.4": { inputUsdPerMillion: 2.5, cachedInputUsdPerMillion: 0.25, outputUsdPerMillion: 15, source: "openai_api_pricing_2026-05-19" }, + "gpt-5.4-mini": { inputUsdPerMillion: 0.75, cachedInputUsdPerMillion: 0.075, outputUsdPerMillion: 4.5, source: "openai_api_pricing_2026-05-19" }, + "gpt-5.4-nano": { inputUsdPerMillion: 0.2, cachedInputUsdPerMillion: 0.02, outputUsdPerMillion: 1.25, source: "openai_api_pricing_2026-05-19" }, + "gpt-5.4-pro": { inputUsdPerMillion: 30, cachedInputUsdPerMillion: null, outputUsdPerMillion: 180, source: "openai_api_pricing_2026-05-19" }, + "gpt-5.3-codex": { inputUsdPerMillion: 1.75, cachedInputUsdPerMillion: 0.175, outputUsdPerMillion: 14, source: "openai_api_pricing_2026-05-19" }, + "gpt-5.2-codex": { inputUsdPerMillion: 1.75, cachedInputUsdPerMillion: 0.175, outputUsdPerMillion: 14, source: "openai_api_pricing_2026-05-19" }, + "gpt-5.2": { inputUsdPerMillion: 1.75, cachedInputUsdPerMillion: 0.175, outputUsdPerMillion: 14, source: "openai_api_pricing_2026-05-19" }, +}; + +function readConfiguredOpenAiRates(): Record { + const raw = process.env.PAPERCLIP_OPENAI_MODEL_RATES_JSON; + if (!raw?.trim()) return {}; + try { + const parsed = JSON.parse(raw) as Record; + const rates: Record = {}; + for (const [model, value] of Object.entries(parsed)) { + if (!model.trim() || typeof value !== "object" || value == null || Array.isArray(value)) continue; + const record = value as Record; + const input = Number(record.inputUsdPerMillion ?? record.input); + const cachedRaw = record.cachedInputUsdPerMillion ?? record.cachedInput ?? record.cached; + const cached = cachedRaw == null ? null : Number(cachedRaw); + const output = Number(record.outputUsdPerMillion ?? record.output); + if (!Number.isFinite(input) || input < 0 || !Number.isFinite(output) || output < 0) continue; + if (cached != null && (!Number.isFinite(cached) || cached < 0)) continue; + rates[model.trim().toLowerCase()] = { + inputUsdPerMillion: input, + cachedInputUsdPerMillion: cached, + outputUsdPerMillion: output, + source: typeof record.source === "string" && record.source.trim() ? record.source.trim() : "env:PAPERCLIP_OPENAI_MODEL_RATES_JSON", + }; + } + return rates; + } catch { + return {}; + } +} + +function normalizeModelId(model: string | null | undefined): string | null { + const normalized = model?.trim().toLowerCase(); + if (!normalized) return null; + return normalized.replace(/-\d{4}-\d{2}-\d{2}$/, ""); +} + +function lookupOpenAiRate(model: string | null | undefined): Rate | null { + const normalized = normalizeModelId(model); + if (!normalized) return null; + const configured = readConfiguredOpenAiRates(); + return configured[normalized] ?? DEFAULT_OPENAI_MODEL_RATES[normalized] ?? null; +} + +function normalizeBillingType(value: AdapterBillingType | BillingType | null | undefined): BillingType { + switch (value) { + case "api": + case "metered_api": + return "metered_api"; + case "subscription": + case "subscription_included": + return "subscription_included"; + case "subscription_overage": + return "subscription_overage"; + case "credits": + return "credits"; + case "fixed": + return "fixed"; + default: + return "unknown"; + } +} + +function hasUsage(usage: UsageSummary | null | undefined): usage is UsageSummary { + return !!usage && ( + usage.inputTokens > 0 || + usage.outputTokens > 0 || + (usage.cachedInputTokens ?? 0) > 0 || + (usage.reasoningOutputTokens ?? 0) > 0 + ); +} + +function estimateOpenAiCostUsd(model: string, usage: UsageSummary): { costUsd: number; rate: Rate } | null { + const rate = lookupOpenAiRate(model); + if (!rate) return null; + + const cachedInputTokens = Math.max(0, Math.floor(usage.cachedInputTokens ?? 0)); + const uncachedInputTokens = Math.max(0, Math.floor(usage.inputTokens) - cachedInputTokens); + const cachedInputRate = rate.cachedInputUsdPerMillion ?? rate.inputUsdPerMillion; + const outputTokens = Math.max(0, Math.floor(usage.outputTokens) + Math.floor(usage.reasoningOutputTokens ?? 0)); + const costUsd = + (uncachedInputTokens * rate.inputUsdPerMillion / 1_000_000) + + (cachedInputTokens * cachedInputRate / 1_000_000) + + (outputTokens * rate.outputUsdPerMillion / 1_000_000); + + return { costUsd, rate }; +} + +export function resolveCostProvenance(result: Pick< + AdapterExecutionResult, + "provider" | "biller" | "model" | "billingType" | "costUsd" | "costSource" | "costMetadata" | "usage" +>): ResolvedCostProvenance { + if (typeof result.costUsd === "number" && Number.isFinite(result.costUsd) && result.costUsd > 0) { + return { + costUsd: result.costUsd, + costSource: result.costSource === "estimated" ? "estimated" : "reported", + costMetadata: { + ...(result.costMetadata ?? {}), + costUsd: result.costUsd, + }, + }; + } + + const provider = result.provider?.trim().toLowerCase(); + const biller = result.biller?.trim().toLowerCase(); + const billingType = normalizeBillingType(result.billingType); + const model = result.model?.trim(); + if (provider === "openai" && biller === "openai" && billingType === "metered_api" && model && hasUsage(result.usage)) { + const estimated = estimateOpenAiCostUsd(model, result.usage); + if (estimated && estimated.costUsd > 0) { + return { + costUsd: estimated.costUsd, + costSource: "estimated", + costMetadata: { + ...(result.costMetadata ?? {}), + estimator: "openai_model_rate_table", + rateSource: estimated.rate.source, + inputUsdPerMillion: estimated.rate.inputUsdPerMillion, + cachedInputUsdPerMillion: estimated.rate.cachedInputUsdPerMillion, + outputUsdPerMillion: estimated.rate.outputUsdPerMillion, + reasoningOutputTokens: result.usage.reasoningOutputTokens ?? 0, + }, + }; + } + } + + return { + costUsd: null, + costSource: "unavailable", + costMetadata: { + ...(result.costMetadata ?? {}), + reason: hasUsage(result.usage) ? "no_reported_cost_or_configured_rate" : "no_token_usage", + }, + }; +}