feat(runner): capture eval usage and estimated cost

packages/paperclip-runner/devtools/issue-thread/src/App.tsx

@@ -77,6 +77,16 @@ interface EmbeddedEvalReport {
     finishedAt: string;
     initialRevision: number;
     finalRevision: number;
+    usage: {
+      agentTurns: number;
+      providerRequests: number | null;
+      inputTokens: number;
+      outputTokens: number;
+      cachedInputTokens: number;
+      reasoningTokens: number;
+      estimatedCostNanodollars: number;
+      pricingVersion: string;
+    } | null;
   };
   view: CapabilityIssueThreadSnapshot;
   devtools: CapabilityDevtoolsSnapshot;

packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx

@@ -13,7 +13,7 @@ export interface EvalInspectorReport {
   disposition: string;
   passed: boolean;
   checks: EvalAssertion[];
-  run: { model: string; provider: string; sessionId: string; fixtureDigest: string; runnerPackageDigest: string; runnerdDigest: string; startedAt: string; finishedAt: string; initialRevision: number; finalRevision: number };
+  run: { model: string; provider: string; sessionId: string; fixtureDigest: string; runnerPackageDigest: string; runnerdDigest: string; startedAt: string; finishedAt: string; initialRevision: number; finalRevision: number; usage: { agentTurns: number; providerRequests: number | null; inputTokens: number; outputTokens: number; cachedInputTokens: number; reasoningTokens: number; estimatedCostNanodollars: number; pricingVersion: string } | null };
 }
 
 function JsonTree({ value, name = "root" }: { value: Json; name?: string }) {
@@ -147,6 +147,10 @@ export function DevtoolsInspector({ snapshot, onFork, tab, onTabChange, evalRepo
             <div><dt>Duration</dt><dd>{Math.max(0, new Date(evalReport.run.finishedAt).getTime() - new Date(evalReport.run.startedAt).getTime())} ms</dd></div>
             <div><dt>Fixture</dt><dd>{evalReport.run.fixtureDigest}</dd></div>
             <div><dt>State</dt><dd>r{evalReport.run.initialRevision} → r{evalReport.run.finalRevision}</dd></div>
+            <div><dt>Tokens</dt><dd>{evalReport.run.usage === null ? "unknown" : `${evalReport.run.usage.inputTokens} in · ${evalReport.run.usage.outputTokens} out · ${evalReport.run.usage.cachedInputTokens} cached`}</dd></div>
+            <div><dt>Agent turns</dt><dd>{evalReport.run.usage?.agentTurns ?? "unknown"}</dd></div>
+            <div><dt>Provider requests</dt><dd>{evalReport.run.usage?.providerRequests ?? "unavailable"}</dd></div>
+            <div><dt>Estimated cost</dt><dd>{evalReport.run.usage === null ? "unknown" : `$${(evalReport.run.usage.estimatedCostNanodollars / 1_000_000_000).toFixed(6)} · ${evalReport.run.usage.pricingVersion}`}</dd></div>
             <div><dt>Runner</dt><dd>{evalReport.run.runnerPackageDigest}</dd></div>
             <div><dt>runnerd</dt><dd>{evalReport.run.runnerdDigest}</dd></div>
           </dl>

packages/paperclip-runner/src/cli/eval-session.ts

@@ -5,6 +5,7 @@ import { resolve } from "node:path";
 
 import { projectCapabilityDevtools } from "../devtools/index.js";
 import { PAPERCLIP_RUNNER_BUILD_METADATA } from "../evals/build-metadata.js";
+import { estimateModelCostNanodollars } from "../evals/model-pricing.js";
 import { projectCapabilityIssueThread } from "../issue-thread/live-projection.js";
 import {
   CapabilityLiveSessionService,
@@ -18,7 +19,7 @@ interface EvalSessionRequest {
   prompt: string;
   model: string;
   runnerd: { path: string; sha256: string };
-  limits: { turnTimeoutMs: number; maxProviderCalls: number; maxCostNanodollars: number };
+  limits: { turnTimeoutMs: number; maxAgentTurns: number; maxEstimatedCostNanodollars: number };
   session: CreateCapabilityLiveSessionInput;
 }
 
@@ -57,9 +58,20 @@ try {
   const turn = await session.sendMessage(request.prompt);
   await session.completeAttempt(turn.status === "completed" ? "succeeded" : "failed", `turn_${turn.status}`);
   const snapshot = session.snapshot();
-  const usage = reconcileCapabilityLiveUsage(snapshot);
-  if (usage.providerCalls > request.limits.maxProviderCalls) throw new Error("provider call limit exceeded");
-  if (usage.costNanodollars > request.limits.maxCostNanodollars) throw new Error("cost limit exceeded");
+  const reconciled = reconcileCapabilityLiveUsage(snapshot);
+  if ((snapshot.usageLedger ?? []).length === 0) throw new Error("completed turn omitted usage accounting");
+  const estimate = estimateModelCostNanodollars(request.model, reconciled);
+  const usage = {
+    agentTurns: reconciled.providerCalls,
+    providerRequests: reconciled.providerRequests,
+    inputTokens: reconciled.inputTokens,
+    outputTokens: reconciled.outputTokens,
+    cachedInputTokens: reconciled.cachedInputTokens,
+    reasoningTokens: reconciled.reasoningTokens,
+    ...estimate,
+  };
+  if (usage.agentTurns > request.limits.maxAgentTurns) throw new Error("agent turn limit exceeded");
+  if (usage.estimatedCostNanodollars > request.limits.maxEstimatedCostNanodollars) throw new Error("estimated cost limit exceeded");
   await writeFile(outputPath, `${JSON.stringify({
     schema: "paperclip-runner/eval-session-artifact/v1",
     attemptId: request.attemptId,

packages/paperclip-runner/src/evals/model-pricing.test.ts

@@ -0,0 +1,25 @@
+import { describe, expect, it } from "vitest";
+
+import { estimateModelCostNanodollars, MODEL_PRICING_VERSION } from "./model-pricing.js";
+
+describe("model pricing", () => {
+  it("prices uncached, cached, and output tokens with explicit provenance", () => {
+    expect(estimateModelCostNanodollars("gpt-5.6-sol", {
+      inputTokens: 1_000,
+      cachedInputTokens: 400,
+      outputTokens: 100,
+    })).toEqual({
+      estimatedCostNanodollars: 6_200_000,
+      pricingVersion: MODEL_PRICING_VERSION,
+      ratesUsdPerMillionTokens: { input: 5, cachedInput: 0.5, output: 30 },
+    });
+  });
+
+  it("fails closed for an unpriced model", () => {
+    expect(() => estimateModelCostNanodollars("unknown", {
+      inputTokens: 1,
+      cachedInputTokens: 0,
+      outputTokens: 1,
+    })).toThrow("model pricing unavailable");
+  });
+});

packages/paperclip-runner/src/evals/model-pricing.ts

@@ -0,0 +1,38 @@
+export const MODEL_PRICING_VERSION = "openai-api-2026-08-20" as const;
+
+interface TokenRatesUsdPerMillion {
+  input: number;
+  cachedInput: number;
+  output: number;
+}
+
+// Versioned OpenAI API list prices, not the customer's actual invoice or Codex-plan debit.
+// Source: https://developers.openai.com/api/docs/models
+const RATES: Readonly<Record<string, TokenRatesUsdPerMillion>> = Object.freeze({
+  "gpt-5.4-mini": { input: 0.75, cachedInput: 0.075, output: 4.5 },
+  "gpt-5.5": { input: 5, cachedInput: 0.5, output: 30 },
+  "gpt-5.6-sol": { input: 5, cachedInput: 0.5, output: 30 },
+  "gpt-5.6-terra": { input: 2, cachedInput: 0.2, output: 12 },
+  "gpt-5.6-luna": { input: 0.2, cachedInput: 0.02, output: 1.2 },
+});
+
+export interface EstimatedModelCost {
+  estimatedCostNanodollars: number;
+  pricingVersion: typeof MODEL_PRICING_VERSION;
+  ratesUsdPerMillionTokens: TokenRatesUsdPerMillion;
+}
+
+export function estimateModelCostNanodollars(
+  model: string,
+  usage: { inputTokens: number; cachedInputTokens: number; outputTokens: number },
+): EstimatedModelCost {
+  const rates = RATES[model];
+  if (rates === undefined) throw new Error(`model pricing unavailable for ${model}`);
+  const uncachedInput = Math.max(0, usage.inputTokens - usage.cachedInputTokens);
+  const estimatedCostNanodollars = Math.round(
+    uncachedInput * rates.input * 1_000
+      + usage.cachedInputTokens * rates.cachedInput * 1_000
+      + usage.outputTokens * rates.output * 1_000,
+  );
+  return { estimatedCostNanodollars, pricingVersion: MODEL_PRICING_VERSION, ratesUsdPerMillionTokens: { ...rates } };
+}

packages/paperclip-runner/src/live/live-session.test.ts

@@ -90,6 +90,14 @@ class FakeCapabilityCodexTransport implements CodexAppServerTransport {
           id: this.state.threadId,
           sessionId: this.state.providerSessionId,
           turns: [...this.state.turns].map(([id, status]) => ({ id, status })),
+          tokenUsage: {
+            total: {
+              inputTokens: this.state.nextTurn * 100,
+              cachedInputTokens: this.state.nextTurn * 20,
+              outputTokens: this.state.nextTurn * 10,
+              reasoningOutputTokens: this.state.nextTurn * 2,
+            },
+          },
         },
       };
     }
@@ -460,10 +468,11 @@ describe("Capability live runnerd and Codex session", () => {
       { attemptId: "attempt-killed", status: "terminated" },
       { attemptId: "attempt-resumed", status: "succeeded" },
     ]);
-    expect(final?.usageLedger).toHaveLength(2);
+    expect(final?.usageLedger).toHaveLength(3);
     expect(final && reconcileCapabilityLiveUsage(final)).toEqual({
-      providerCalls: 2,
-      inputTokens: 180,
+      providerCalls: 3,
+      providerRequests: 0,
+      inputTokens: 280,
       outputTokens: 30,
       cachedInputTokens: 40,
       reasoningTokens: 5,
@@ -712,7 +721,7 @@ describe("Capability live runnerd and Codex session", () => {
         { attemptId: "attempt-real-killed", status: "terminated" },
         { attemptId: "attempt-real-resumed", status: "succeeded", resumeOf: "attempt-real-killed" },
       ]);
-      expect(final?.usageLedger).toHaveLength(1);
+      expect(final?.usageLedger).toHaveLength(2);
       expect(final?.terminalTurns).toEqual(expect.arrayContaining([
         expect.objectContaining({ turnId: "turn-1", status: "interrupted" }),
         expect.objectContaining({ turnId: "turn-2", status: "completed" }),

packages/paperclip-runner/src/live/live-session.ts

@@ -121,6 +121,7 @@ export interface CapabilityLiveUsageReceipt {
   providerResponseId: string | null;
   turnId: string | null;
   providerCalls: number;
+  providerRequests: number;
   inputTokens: number;
   outputTokens: number;
   cachedInputTokens: number;
@@ -142,6 +143,7 @@ export interface RecordCapabilityLiveUsageInput {
   providerResponseId?: string | null;
   turnId?: string | null;
   providerCalls?: number;
+  providerRequests?: number;
   inputTokens?: number;
   outputTokens?: number;
   cachedInputTokens?: number;
@@ -462,6 +464,7 @@ export function assertCapabilityLiveSessionSnapshot(
     }
     for (const name of [
       "providerCalls",
+      "providerRequests",
       "inputTokens",
       "outputTokens",
       "cachedInputTokens",
@@ -506,6 +509,7 @@ export function reconcileCapabilityLiveUsage(
   const unique = new Map((snapshot.usageLedger ?? []).map((receipt) => [receipt.receiptId, receipt]));
   return [...unique.values()].reduce((total, receipt) => ({
     providerCalls: total.providerCalls + receipt.providerCalls,
+    providerRequests: total.providerRequests + receipt.providerRequests,
     inputTokens: total.inputTokens + receipt.inputTokens,
     outputTokens: total.outputTokens + receipt.outputTokens,
     cachedInputTokens: total.cachedInputTokens + receipt.cachedInputTokens,
@@ -513,6 +517,7 @@ export function reconcileCapabilityLiveUsage(
     costNanodollars: total.costNanodollars + receipt.costNanodollars,
   }), {
     providerCalls: 0,
+    providerRequests: 0,
     inputTokens: 0,
     outputTokens: 0,
     cachedInputTokens: 0,
@@ -804,6 +809,7 @@ export class CapabilityLiveSession {
   readonly #listeners = new Set<CapabilityLiveTurnListener>();
   #eventSeq = 0;
   #turnEventCount = 0;
+  readonly #rawUsageByTurn = new Map<string, { providerRequests: number; inputTokens: number; outputTokens: number; cachedInputTokens: number; reasoningTokens: number }>();
 
   private constructor(options: OpenSessionOptions & { snapshot?: CapabilityLiveSessionSnapshot }) {
     this.#port = options.port;
@@ -1020,6 +1026,7 @@ export class CapabilityLiveSession {
       providerResponseId: input.providerResponseId ?? null,
       turnId: input.turnId ?? this.#activeTurnId,
       providerCalls: nonNegativeInteger(input.providerCalls ?? 1, "provider_calls"),
+      providerRequests: nonNegativeInteger(input.providerRequests, "provider_requests"),
       inputTokens: nonNegativeInteger(input.inputTokens, "input_tokens"),
       outputTokens: nonNegativeInteger(input.outputTokens, "output_tokens"),
       cachedInputTokens: nonNegativeInteger(input.cachedInputTokens, "cached_input_tokens"),
@@ -1112,10 +1119,51 @@ export class CapabilityLiveSession {
     if (draft !== undefined && draft.turnId === null) draft.turnId = turnId;
     await this.#persist();
     const result = await terminal;
+    await this.#captureTurnUsage(result.turnId);
     await this.#persist();
     return { ...result, snapshot: this.snapshot() };
   }
 
+  async #captureTurnUsage(turnId: string): Promise<void> {
+    if (this.#transport === null) throw new Error("capability_live_usage_transport_missing");
+    const captured = this.#rawUsageByTurn.get(turnId);
+    const read = captured === undefined ? await this.#transport.request("thread/read", {
+      threadId: this.#providerThreadId,
+      includeTurns: true,
+    }) : {};
+    const thread = record(read.thread);
+    const usage = record(thread.tokenUsage ?? read.tokenUsage);
+    const total = record(captured ?? usage.total ?? usage.totalTokenUsage ?? usage.total_token_usage);
+    const integer = (...names: string[]): number => {
+      for (const name of names) {
+        const value = total[name];
+        if (Number.isSafeInteger(value) && Number(value) >= 0) return Number(value);
+      }
+      return 0;
+    };
+    const cumulative = {
+      inputTokens: integer("inputTokens", "input_tokens"),
+      outputTokens: integer("outputTokens", "output_tokens"),
+      cachedInputTokens: integer("cachedInputTokens", "cached_input_tokens"),
+      reasoningTokens: integer("reasoningOutputTokens", "reasoningTokens", "reasoning_output_tokens"),
+    };
+    if (cumulative.inputTokens + cumulative.outputTokens === 0) {
+      throw new Error("capability_live_usage_missing");
+    }
+    const prior = reconcileCapabilityLiveUsage(this.snapshot());
+    await this.recordUsage({
+      receiptId: `${turnId}:usage`,
+      turnId,
+      providerCalls: 1,
+      providerRequests: captured?.providerRequests ?? 0,
+      inputTokens: Math.max(0, cumulative.inputTokens - prior.inputTokens),
+      outputTokens: Math.max(0, cumulative.outputTokens - prior.outputTokens),
+      cachedInputTokens: Math.max(0, cumulative.cachedInputTokens - prior.cachedInputTokens),
+      reasoningTokens: Math.max(0, cumulative.reasoningTokens - prior.reasoningTokens),
+      costNanodollars: 0,
+    });
+  }
+
   /**
    * Runs an operator-authored DevTools call through the exact dispatcher used
    * by Codex. It receives a real turn id, durable call/result evidence, and a
@@ -1380,7 +1428,7 @@ export class CapabilityLiveSession {
         approvalPolicy: "never",
         baseInstructions: LIVE_BASE_INSTRUCTIONS,
         dynamicTools: tools.map(dynamicToolSpec),
-        experimentalRawEvents: false,
+        experimentalRawEvents: true,
         persistExtendedHistory: true,
       });
       const thread = record(opened.thread);
@@ -1565,7 +1613,20 @@ export class CapabilityLiveSession {
       method: notification.method,
       params: jsonValue(params),
     });
-    if (notification.method === "turn/started") {
+    if (notification.method === "rawResponse/completed") {
+      const rawUsage = record(params.usage);
+      if (turnId.length > 0 && Object.keys(rawUsage).length > 0) {
+        const previous = this.#rawUsageByTurn.get(turnId) ?? { providerRequests: 0, inputTokens: 0, outputTokens: 0, cachedInputTokens: 0, reasoningTokens: 0 };
+        const value = (name: string): number => Number.isSafeInteger(rawUsage[name]) && Number(rawUsage[name]) >= 0 ? Number(rawUsage[name]) : 0;
+        this.#rawUsageByTurn.set(turnId, {
+          providerRequests: previous.providerRequests + 1,
+          inputTokens: previous.inputTokens + value("inputTokens"),
+          outputTokens: previous.outputTokens + value("outputTokens"),
+          cachedInputTokens: previous.cachedInputTokens + value("cachedInputTokens"),
+          reasoningTokens: previous.reasoningTokens + value("reasoningOutputTokens"),
+        });
+      }
+    } else if (notification.method === "turn/started") {
       if (turnId.length > 0) this.#activeTurnId = turnId;
       this.#emit({ turnId: turnId || this.#activeTurnId, kind: "activity", reason: "turn_started" });
     } else if (notification.method === "item/agentMessage/delta") {

packages/paperclip-runner/test/fixtures/fake-durable-codex-app-server.mjs

@@ -74,6 +74,14 @@ function handleRequest(message) {
           id: state.threadId,
           sessionId: state.sessionId,
           turns: Object.entries(state.turns).map(([turnId, status]) => ({ id: turnId, status })),
+          tokenUsage: {
+            total: {
+              inputTokens: state.nextTurn * 10,
+              cachedInputTokens: 0,
+              outputTokens: state.nextTurn * 2,
+              reasoningOutputTokens: 0,
+            },
+          },
         },
       },
     });