diff --git a/packages/paperclip-runner/devtools/issue-thread/src/App.tsx b/packages/paperclip-runner/devtools/issue-thread/src/App.tsx index 49b128e7a96..f4b58de8040 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/App.tsx +++ b/packages/paperclip-runner/devtools/issue-thread/src/App.tsx @@ -77,6 +77,16 @@ interface EmbeddedEvalReport { finishedAt: string; initialRevision: number; finalRevision: number; + usage: { + agentTurns: number; + providerRequests: number | null; + inputTokens: number; + outputTokens: number; + cachedInputTokens: number; + reasoningTokens: number; + estimatedCostNanodollars: number; + pricingVersion: string; + } | null; }; view: CapabilityIssueThreadSnapshot; devtools: CapabilityDevtoolsSnapshot; diff --git a/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx b/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx index d795d94a3e2..84c0581025d 100644 --- a/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx +++ b/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx @@ -13,7 +13,7 @@ export interface EvalInspectorReport { disposition: string; passed: boolean; checks: EvalAssertion[]; - run: { model: string; provider: string; sessionId: string; fixtureDigest: string; runnerPackageDigest: string; runnerdDigest: string; startedAt: string; finishedAt: string; initialRevision: number; finalRevision: number }; + run: { model: string; provider: string; sessionId: string; fixtureDigest: string; runnerPackageDigest: string; runnerdDigest: string; startedAt: string; finishedAt: string; initialRevision: number; finalRevision: number; usage: { agentTurns: number; providerRequests: number | null; inputTokens: number; outputTokens: number; cachedInputTokens: number; reasoningTokens: number; estimatedCostNanodollars: number; pricingVersion: string } | null }; } function JsonTree({ value, name = "root" }: { value: Json; name?: string }) { @@ -147,6 +147,10 @@ export function DevtoolsInspector({ snapshot, onFork, tab, onTabChange, evalRepo
Duration
{Math.max(0, new Date(evalReport.run.finishedAt).getTime() - new Date(evalReport.run.startedAt).getTime())} ms
Fixture
{evalReport.run.fixtureDigest}
State
r{evalReport.run.initialRevision} → r{evalReport.run.finalRevision}
+
Tokens
{evalReport.run.usage === null ? "unknown" : `${evalReport.run.usage.inputTokens} in · ${evalReport.run.usage.outputTokens} out · ${evalReport.run.usage.cachedInputTokens} cached`}
+
Agent turns
{evalReport.run.usage?.agentTurns ?? "unknown"}
+
Provider requests
{evalReport.run.usage?.providerRequests ?? "unavailable"}
+
Estimated cost
{evalReport.run.usage === null ? "unknown" : `$${(evalReport.run.usage.estimatedCostNanodollars / 1_000_000_000).toFixed(6)} · ${evalReport.run.usage.pricingVersion}`}
Runner
{evalReport.run.runnerPackageDigest}
runnerd
{evalReport.run.runnerdDigest}
diff --git a/packages/paperclip-runner/src/cli/eval-session.ts b/packages/paperclip-runner/src/cli/eval-session.ts index d2da0cf5e0e..2a3f4b19b06 100644 --- a/packages/paperclip-runner/src/cli/eval-session.ts +++ b/packages/paperclip-runner/src/cli/eval-session.ts @@ -5,6 +5,7 @@ import { resolve } from "node:path"; import { projectCapabilityDevtools } from "../devtools/index.js"; import { PAPERCLIP_RUNNER_BUILD_METADATA } from "../evals/build-metadata.js"; +import { estimateModelCostNanodollars } from "../evals/model-pricing.js"; import { projectCapabilityIssueThread } from "../issue-thread/live-projection.js"; import { CapabilityLiveSessionService, @@ -18,7 +19,7 @@ interface EvalSessionRequest { prompt: string; model: string; runnerd: { path: string; sha256: string }; - limits: { turnTimeoutMs: number; maxProviderCalls: number; maxCostNanodollars: number }; + limits: { turnTimeoutMs: number; maxAgentTurns: number; maxEstimatedCostNanodollars: number }; session: CreateCapabilityLiveSessionInput; } @@ -57,9 +58,20 @@ try { const turn = await session.sendMessage(request.prompt); await session.completeAttempt(turn.status === "completed" ? "succeeded" : "failed", `turn_${turn.status}`); const snapshot = session.snapshot(); - const usage = reconcileCapabilityLiveUsage(snapshot); - if (usage.providerCalls > request.limits.maxProviderCalls) throw new Error("provider call limit exceeded"); - if (usage.costNanodollars > request.limits.maxCostNanodollars) throw new Error("cost limit exceeded"); + const reconciled = reconcileCapabilityLiveUsage(snapshot); + if ((snapshot.usageLedger ?? []).length === 0) throw new Error("completed turn omitted usage accounting"); + const estimate = estimateModelCostNanodollars(request.model, reconciled); + const usage = { + agentTurns: reconciled.providerCalls, + providerRequests: reconciled.providerRequests, + inputTokens: reconciled.inputTokens, + outputTokens: reconciled.outputTokens, + cachedInputTokens: reconciled.cachedInputTokens, + reasoningTokens: reconciled.reasoningTokens, + ...estimate, + }; + if (usage.agentTurns > request.limits.maxAgentTurns) throw new Error("agent turn limit exceeded"); + if (usage.estimatedCostNanodollars > request.limits.maxEstimatedCostNanodollars) throw new Error("estimated cost limit exceeded"); await writeFile(outputPath, `${JSON.stringify({ schema: "paperclip-runner/eval-session-artifact/v1", attemptId: request.attemptId, diff --git a/packages/paperclip-runner/src/evals/model-pricing.test.ts b/packages/paperclip-runner/src/evals/model-pricing.test.ts new file mode 100644 index 00000000000..1fc15230a24 --- /dev/null +++ b/packages/paperclip-runner/src/evals/model-pricing.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; + +import { estimateModelCostNanodollars, MODEL_PRICING_VERSION } from "./model-pricing.js"; + +describe("model pricing", () => { + it("prices uncached, cached, and output tokens with explicit provenance", () => { + expect(estimateModelCostNanodollars("gpt-5.6-sol", { + inputTokens: 1_000, + cachedInputTokens: 400, + outputTokens: 100, + })).toEqual({ + estimatedCostNanodollars: 6_200_000, + pricingVersion: MODEL_PRICING_VERSION, + ratesUsdPerMillionTokens: { input: 5, cachedInput: 0.5, output: 30 }, + }); + }); + + it("fails closed for an unpriced model", () => { + expect(() => estimateModelCostNanodollars("unknown", { + inputTokens: 1, + cachedInputTokens: 0, + outputTokens: 1, + })).toThrow("model pricing unavailable"); + }); +}); diff --git a/packages/paperclip-runner/src/evals/model-pricing.ts b/packages/paperclip-runner/src/evals/model-pricing.ts new file mode 100644 index 00000000000..c680be4077b --- /dev/null +++ b/packages/paperclip-runner/src/evals/model-pricing.ts @@ -0,0 +1,38 @@ +export const MODEL_PRICING_VERSION = "openai-api-2026-08-20" as const; + +interface TokenRatesUsdPerMillion { + input: number; + cachedInput: number; + output: number; +} + +// Versioned OpenAI API list prices, not the customer's actual invoice or Codex-plan debit. +// Source: https://developers.openai.com/api/docs/models +const RATES: Readonly> = Object.freeze({ + "gpt-5.4-mini": { input: 0.75, cachedInput: 0.075, output: 4.5 }, + "gpt-5.5": { input: 5, cachedInput: 0.5, output: 30 }, + "gpt-5.6-sol": { input: 5, cachedInput: 0.5, output: 30 }, + "gpt-5.6-terra": { input: 2, cachedInput: 0.2, output: 12 }, + "gpt-5.6-luna": { input: 0.2, cachedInput: 0.02, output: 1.2 }, +}); + +export interface EstimatedModelCost { + estimatedCostNanodollars: number; + pricingVersion: typeof MODEL_PRICING_VERSION; + ratesUsdPerMillionTokens: TokenRatesUsdPerMillion; +} + +export function estimateModelCostNanodollars( + model: string, + usage: { inputTokens: number; cachedInputTokens: number; outputTokens: number }, +): EstimatedModelCost { + const rates = RATES[model]; + if (rates === undefined) throw new Error(`model pricing unavailable for ${model}`); + const uncachedInput = Math.max(0, usage.inputTokens - usage.cachedInputTokens); + const estimatedCostNanodollars = Math.round( + uncachedInput * rates.input * 1_000 + + usage.cachedInputTokens * rates.cachedInput * 1_000 + + usage.outputTokens * rates.output * 1_000, + ); + return { estimatedCostNanodollars, pricingVersion: MODEL_PRICING_VERSION, ratesUsdPerMillionTokens: { ...rates } }; +} diff --git a/packages/paperclip-runner/src/live/live-session.test.ts b/packages/paperclip-runner/src/live/live-session.test.ts index 7e3e4413e26..12c39b4abff 100644 --- a/packages/paperclip-runner/src/live/live-session.test.ts +++ b/packages/paperclip-runner/src/live/live-session.test.ts @@ -90,6 +90,14 @@ class FakeCapabilityCodexTransport implements CodexAppServerTransport { id: this.state.threadId, sessionId: this.state.providerSessionId, turns: [...this.state.turns].map(([id, status]) => ({ id, status })), + tokenUsage: { + total: { + inputTokens: this.state.nextTurn * 100, + cachedInputTokens: this.state.nextTurn * 20, + outputTokens: this.state.nextTurn * 10, + reasoningOutputTokens: this.state.nextTurn * 2, + }, + }, }, }; } @@ -460,10 +468,11 @@ describe("Capability live runnerd and Codex session", () => { { attemptId: "attempt-killed", status: "terminated" }, { attemptId: "attempt-resumed", status: "succeeded" }, ]); - expect(final?.usageLedger).toHaveLength(2); + expect(final?.usageLedger).toHaveLength(3); expect(final && reconcileCapabilityLiveUsage(final)).toEqual({ - providerCalls: 2, - inputTokens: 180, + providerCalls: 3, + providerRequests: 0, + inputTokens: 280, outputTokens: 30, cachedInputTokens: 40, reasoningTokens: 5, @@ -712,7 +721,7 @@ describe("Capability live runnerd and Codex session", () => { { attemptId: "attempt-real-killed", status: "terminated" }, { attemptId: "attempt-real-resumed", status: "succeeded", resumeOf: "attempt-real-killed" }, ]); - expect(final?.usageLedger).toHaveLength(1); + expect(final?.usageLedger).toHaveLength(2); expect(final?.terminalTurns).toEqual(expect.arrayContaining([ expect.objectContaining({ turnId: "turn-1", status: "interrupted" }), expect.objectContaining({ turnId: "turn-2", status: "completed" }), diff --git a/packages/paperclip-runner/src/live/live-session.ts b/packages/paperclip-runner/src/live/live-session.ts index 48de2bc6eff..6207c009731 100644 --- a/packages/paperclip-runner/src/live/live-session.ts +++ b/packages/paperclip-runner/src/live/live-session.ts @@ -121,6 +121,7 @@ export interface CapabilityLiveUsageReceipt { providerResponseId: string | null; turnId: string | null; providerCalls: number; + providerRequests: number; inputTokens: number; outputTokens: number; cachedInputTokens: number; @@ -142,6 +143,7 @@ export interface RecordCapabilityLiveUsageInput { providerResponseId?: string | null; turnId?: string | null; providerCalls?: number; + providerRequests?: number; inputTokens?: number; outputTokens?: number; cachedInputTokens?: number; @@ -462,6 +464,7 @@ export function assertCapabilityLiveSessionSnapshot( } for (const name of [ "providerCalls", + "providerRequests", "inputTokens", "outputTokens", "cachedInputTokens", @@ -506,6 +509,7 @@ export function reconcileCapabilityLiveUsage( const unique = new Map((snapshot.usageLedger ?? []).map((receipt) => [receipt.receiptId, receipt])); return [...unique.values()].reduce((total, receipt) => ({ providerCalls: total.providerCalls + receipt.providerCalls, + providerRequests: total.providerRequests + receipt.providerRequests, inputTokens: total.inputTokens + receipt.inputTokens, outputTokens: total.outputTokens + receipt.outputTokens, cachedInputTokens: total.cachedInputTokens + receipt.cachedInputTokens, @@ -513,6 +517,7 @@ export function reconcileCapabilityLiveUsage( costNanodollars: total.costNanodollars + receipt.costNanodollars, }), { providerCalls: 0, + providerRequests: 0, inputTokens: 0, outputTokens: 0, cachedInputTokens: 0, @@ -804,6 +809,7 @@ export class CapabilityLiveSession { readonly #listeners = new Set(); #eventSeq = 0; #turnEventCount = 0; + readonly #rawUsageByTurn = new Map(); private constructor(options: OpenSessionOptions & { snapshot?: CapabilityLiveSessionSnapshot }) { this.#port = options.port; @@ -1020,6 +1026,7 @@ export class CapabilityLiveSession { providerResponseId: input.providerResponseId ?? null, turnId: input.turnId ?? this.#activeTurnId, providerCalls: nonNegativeInteger(input.providerCalls ?? 1, "provider_calls"), + providerRequests: nonNegativeInteger(input.providerRequests, "provider_requests"), inputTokens: nonNegativeInteger(input.inputTokens, "input_tokens"), outputTokens: nonNegativeInteger(input.outputTokens, "output_tokens"), cachedInputTokens: nonNegativeInteger(input.cachedInputTokens, "cached_input_tokens"), @@ -1112,10 +1119,51 @@ export class CapabilityLiveSession { if (draft !== undefined && draft.turnId === null) draft.turnId = turnId; await this.#persist(); const result = await terminal; + await this.#captureTurnUsage(result.turnId); await this.#persist(); return { ...result, snapshot: this.snapshot() }; } + async #captureTurnUsage(turnId: string): Promise { + if (this.#transport === null) throw new Error("capability_live_usage_transport_missing"); + const captured = this.#rawUsageByTurn.get(turnId); + const read = captured === undefined ? await this.#transport.request("thread/read", { + threadId: this.#providerThreadId, + includeTurns: true, + }) : {}; + const thread = record(read.thread); + const usage = record(thread.tokenUsage ?? read.tokenUsage); + const total = record(captured ?? usage.total ?? usage.totalTokenUsage ?? usage.total_token_usage); + const integer = (...names: string[]): number => { + for (const name of names) { + const value = total[name]; + if (Number.isSafeInteger(value) && Number(value) >= 0) return Number(value); + } + return 0; + }; + const cumulative = { + inputTokens: integer("inputTokens", "input_tokens"), + outputTokens: integer("outputTokens", "output_tokens"), + cachedInputTokens: integer("cachedInputTokens", "cached_input_tokens"), + reasoningTokens: integer("reasoningOutputTokens", "reasoningTokens", "reasoning_output_tokens"), + }; + if (cumulative.inputTokens + cumulative.outputTokens === 0) { + throw new Error("capability_live_usage_missing"); + } + const prior = reconcileCapabilityLiveUsage(this.snapshot()); + await this.recordUsage({ + receiptId: `${turnId}:usage`, + turnId, + providerCalls: 1, + providerRequests: captured?.providerRequests ?? 0, + inputTokens: Math.max(0, cumulative.inputTokens - prior.inputTokens), + outputTokens: Math.max(0, cumulative.outputTokens - prior.outputTokens), + cachedInputTokens: Math.max(0, cumulative.cachedInputTokens - prior.cachedInputTokens), + reasoningTokens: Math.max(0, cumulative.reasoningTokens - prior.reasoningTokens), + costNanodollars: 0, + }); + } + /** * Runs an operator-authored DevTools call through the exact dispatcher used * by Codex. It receives a real turn id, durable call/result evidence, and a @@ -1380,7 +1428,7 @@ export class CapabilityLiveSession { approvalPolicy: "never", baseInstructions: LIVE_BASE_INSTRUCTIONS, dynamicTools: tools.map(dynamicToolSpec), - experimentalRawEvents: false, + experimentalRawEvents: true, persistExtendedHistory: true, }); const thread = record(opened.thread); @@ -1565,7 +1613,20 @@ export class CapabilityLiveSession { method: notification.method, params: jsonValue(params), }); - if (notification.method === "turn/started") { + if (notification.method === "rawResponse/completed") { + const rawUsage = record(params.usage); + if (turnId.length > 0 && Object.keys(rawUsage).length > 0) { + const previous = this.#rawUsageByTurn.get(turnId) ?? { providerRequests: 0, inputTokens: 0, outputTokens: 0, cachedInputTokens: 0, reasoningTokens: 0 }; + const value = (name: string): number => Number.isSafeInteger(rawUsage[name]) && Number(rawUsage[name]) >= 0 ? Number(rawUsage[name]) : 0; + this.#rawUsageByTurn.set(turnId, { + providerRequests: previous.providerRequests + 1, + inputTokens: previous.inputTokens + value("inputTokens"), + outputTokens: previous.outputTokens + value("outputTokens"), + cachedInputTokens: previous.cachedInputTokens + value("cachedInputTokens"), + reasoningTokens: previous.reasoningTokens + value("reasoningOutputTokens"), + }); + } + } else if (notification.method === "turn/started") { if (turnId.length > 0) this.#activeTurnId = turnId; this.#emit({ turnId: turnId || this.#activeTurnId, kind: "activity", reason: "turn_started" }); } else if (notification.method === "item/agentMessage/delta") { diff --git a/packages/paperclip-runner/test/fixtures/fake-durable-codex-app-server.mjs b/packages/paperclip-runner/test/fixtures/fake-durable-codex-app-server.mjs index f343e40ad16..0e744f37cab 100644 --- a/packages/paperclip-runner/test/fixtures/fake-durable-codex-app-server.mjs +++ b/packages/paperclip-runner/test/fixtures/fake-durable-codex-app-server.mjs @@ -74,6 +74,14 @@ function handleRequest(message) { id: state.threadId, sessionId: state.sessionId, turns: Object.entries(state.turns).map(([turnId, status]) => ({ id: turnId, status })), + tokenUsage: { + total: { + inputTokens: state.nextTurn * 10, + cachedInputTokens: 0, + outputTokens: state.nextTurn * 2, + reasoningOutputTokens: 0, + }, + }, }, }, });