diff --git a/packages/paperclip-runner/devtools/issue-thread/src/App.tsx b/packages/paperclip-runner/devtools/issue-thread/src/App.tsx
index 49b128e7a96..f4b58de8040 100644
--- a/packages/paperclip-runner/devtools/issue-thread/src/App.tsx
+++ b/packages/paperclip-runner/devtools/issue-thread/src/App.tsx
@@ -77,6 +77,16 @@ interface EmbeddedEvalReport {
finishedAt: string;
initialRevision: number;
finalRevision: number;
+ usage: {
+ agentTurns: number;
+ providerRequests: number | null;
+ inputTokens: number;
+ outputTokens: number;
+ cachedInputTokens: number;
+ reasoningTokens: number;
+ estimatedCostNanodollars: number;
+ pricingVersion: string;
+ } | null;
};
view: CapabilityIssueThreadSnapshot;
devtools: CapabilityDevtoolsSnapshot;
diff --git a/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx b/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx
index d795d94a3e2..84c0581025d 100644
--- a/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx
+++ b/packages/paperclip-runner/devtools/issue-thread/src/DevtoolsInspector.tsx
@@ -13,7 +13,7 @@ export interface EvalInspectorReport {
disposition: string;
passed: boolean;
checks: EvalAssertion[];
- run: { model: string; provider: string; sessionId: string; fixtureDigest: string; runnerPackageDigest: string; runnerdDigest: string; startedAt: string; finishedAt: string; initialRevision: number; finalRevision: number };
+ run: { model: string; provider: string; sessionId: string; fixtureDigest: string; runnerPackageDigest: string; runnerdDigest: string; startedAt: string; finishedAt: string; initialRevision: number; finalRevision: number; usage: { agentTurns: number; providerRequests: number | null; inputTokens: number; outputTokens: number; cachedInputTokens: number; reasoningTokens: number; estimatedCostNanodollars: number; pricingVersion: string } | null };
}
function JsonTree({ value, name = "root" }: { value: Json; name?: string }) {
@@ -147,6 +147,10 @@ export function DevtoolsInspector({ snapshot, onFork, tab, onTabChange, evalRepo
Duration{Math.max(0, new Date(evalReport.run.finishedAt).getTime() - new Date(evalReport.run.startedAt).getTime())} ms
Fixture{evalReport.run.fixtureDigest}
Stater{evalReport.run.initialRevision} → r{evalReport.run.finalRevision}
+ Tokens{evalReport.run.usage === null ? "unknown" : `${evalReport.run.usage.inputTokens} in · ${evalReport.run.usage.outputTokens} out · ${evalReport.run.usage.cachedInputTokens} cached`}
+ Agent turns{evalReport.run.usage?.agentTurns ?? "unknown"}
+ Provider requests{evalReport.run.usage?.providerRequests ?? "unavailable"}
+ Estimated cost{evalReport.run.usage === null ? "unknown" : `$${(evalReport.run.usage.estimatedCostNanodollars / 1_000_000_000).toFixed(6)} · ${evalReport.run.usage.pricingVersion}`}
Runner{evalReport.run.runnerPackageDigest}
runnerd{evalReport.run.runnerdDigest}
diff --git a/packages/paperclip-runner/src/cli/eval-session.ts b/packages/paperclip-runner/src/cli/eval-session.ts
index d2da0cf5e0e..2a3f4b19b06 100644
--- a/packages/paperclip-runner/src/cli/eval-session.ts
+++ b/packages/paperclip-runner/src/cli/eval-session.ts
@@ -5,6 +5,7 @@ import { resolve } from "node:path";
import { projectCapabilityDevtools } from "../devtools/index.js";
import { PAPERCLIP_RUNNER_BUILD_METADATA } from "../evals/build-metadata.js";
+import { estimateModelCostNanodollars } from "../evals/model-pricing.js";
import { projectCapabilityIssueThread } from "../issue-thread/live-projection.js";
import {
CapabilityLiveSessionService,
@@ -18,7 +19,7 @@ interface EvalSessionRequest {
prompt: string;
model: string;
runnerd: { path: string; sha256: string };
- limits: { turnTimeoutMs: number; maxProviderCalls: number; maxCostNanodollars: number };
+ limits: { turnTimeoutMs: number; maxAgentTurns: number; maxEstimatedCostNanodollars: number };
session: CreateCapabilityLiveSessionInput;
}
@@ -57,9 +58,20 @@ try {
const turn = await session.sendMessage(request.prompt);
await session.completeAttempt(turn.status === "completed" ? "succeeded" : "failed", `turn_${turn.status}`);
const snapshot = session.snapshot();
- const usage = reconcileCapabilityLiveUsage(snapshot);
- if (usage.providerCalls > request.limits.maxProviderCalls) throw new Error("provider call limit exceeded");
- if (usage.costNanodollars > request.limits.maxCostNanodollars) throw new Error("cost limit exceeded");
+ const reconciled = reconcileCapabilityLiveUsage(snapshot);
+ if ((snapshot.usageLedger ?? []).length === 0) throw new Error("completed turn omitted usage accounting");
+ const estimate = estimateModelCostNanodollars(request.model, reconciled);
+ const usage = {
+ agentTurns: reconciled.providerCalls,
+ providerRequests: reconciled.providerRequests,
+ inputTokens: reconciled.inputTokens,
+ outputTokens: reconciled.outputTokens,
+ cachedInputTokens: reconciled.cachedInputTokens,
+ reasoningTokens: reconciled.reasoningTokens,
+ ...estimate,
+ };
+ if (usage.agentTurns > request.limits.maxAgentTurns) throw new Error("agent turn limit exceeded");
+ if (usage.estimatedCostNanodollars > request.limits.maxEstimatedCostNanodollars) throw new Error("estimated cost limit exceeded");
await writeFile(outputPath, `${JSON.stringify({
schema: "paperclip-runner/eval-session-artifact/v1",
attemptId: request.attemptId,
diff --git a/packages/paperclip-runner/src/evals/model-pricing.test.ts b/packages/paperclip-runner/src/evals/model-pricing.test.ts
new file mode 100644
index 00000000000..1fc15230a24
--- /dev/null
+++ b/packages/paperclip-runner/src/evals/model-pricing.test.ts
@@ -0,0 +1,25 @@
+import { describe, expect, it } from "vitest";
+
+import { estimateModelCostNanodollars, MODEL_PRICING_VERSION } from "./model-pricing.js";
+
+describe("model pricing", () => {
+ it("prices uncached, cached, and output tokens with explicit provenance", () => {
+ expect(estimateModelCostNanodollars("gpt-5.6-sol", {
+ inputTokens: 1_000,
+ cachedInputTokens: 400,
+ outputTokens: 100,
+ })).toEqual({
+ estimatedCostNanodollars: 6_200_000,
+ pricingVersion: MODEL_PRICING_VERSION,
+ ratesUsdPerMillionTokens: { input: 5, cachedInput: 0.5, output: 30 },
+ });
+ });
+
+ it("fails closed for an unpriced model", () => {
+ expect(() => estimateModelCostNanodollars("unknown", {
+ inputTokens: 1,
+ cachedInputTokens: 0,
+ outputTokens: 1,
+ })).toThrow("model pricing unavailable");
+ });
+});
diff --git a/packages/paperclip-runner/src/evals/model-pricing.ts b/packages/paperclip-runner/src/evals/model-pricing.ts
new file mode 100644
index 00000000000..c680be4077b
--- /dev/null
+++ b/packages/paperclip-runner/src/evals/model-pricing.ts
@@ -0,0 +1,38 @@
+export const MODEL_PRICING_VERSION = "openai-api-2026-08-20" as const;
+
+interface TokenRatesUsdPerMillion {
+ input: number;
+ cachedInput: number;
+ output: number;
+}
+
+// Versioned OpenAI API list prices, not the customer's actual invoice or Codex-plan debit.
+// Source: https://developers.openai.com/api/docs/models
+const RATES: Readonly> = Object.freeze({
+ "gpt-5.4-mini": { input: 0.75, cachedInput: 0.075, output: 4.5 },
+ "gpt-5.5": { input: 5, cachedInput: 0.5, output: 30 },
+ "gpt-5.6-sol": { input: 5, cachedInput: 0.5, output: 30 },
+ "gpt-5.6-terra": { input: 2, cachedInput: 0.2, output: 12 },
+ "gpt-5.6-luna": { input: 0.2, cachedInput: 0.02, output: 1.2 },
+});
+
+export interface EstimatedModelCost {
+ estimatedCostNanodollars: number;
+ pricingVersion: typeof MODEL_PRICING_VERSION;
+ ratesUsdPerMillionTokens: TokenRatesUsdPerMillion;
+}
+
+export function estimateModelCostNanodollars(
+ model: string,
+ usage: { inputTokens: number; cachedInputTokens: number; outputTokens: number },
+): EstimatedModelCost {
+ const rates = RATES[model];
+ if (rates === undefined) throw new Error(`model pricing unavailable for ${model}`);
+ const uncachedInput = Math.max(0, usage.inputTokens - usage.cachedInputTokens);
+ const estimatedCostNanodollars = Math.round(
+ uncachedInput * rates.input * 1_000
+ + usage.cachedInputTokens * rates.cachedInput * 1_000
+ + usage.outputTokens * rates.output * 1_000,
+ );
+ return { estimatedCostNanodollars, pricingVersion: MODEL_PRICING_VERSION, ratesUsdPerMillionTokens: { ...rates } };
+}
diff --git a/packages/paperclip-runner/src/live/live-session.test.ts b/packages/paperclip-runner/src/live/live-session.test.ts
index 7e3e4413e26..12c39b4abff 100644
--- a/packages/paperclip-runner/src/live/live-session.test.ts
+++ b/packages/paperclip-runner/src/live/live-session.test.ts
@@ -90,6 +90,14 @@ class FakeCapabilityCodexTransport implements CodexAppServerTransport {
id: this.state.threadId,
sessionId: this.state.providerSessionId,
turns: [...this.state.turns].map(([id, status]) => ({ id, status })),
+ tokenUsage: {
+ total: {
+ inputTokens: this.state.nextTurn * 100,
+ cachedInputTokens: this.state.nextTurn * 20,
+ outputTokens: this.state.nextTurn * 10,
+ reasoningOutputTokens: this.state.nextTurn * 2,
+ },
+ },
},
};
}
@@ -460,10 +468,11 @@ describe("Capability live runnerd and Codex session", () => {
{ attemptId: "attempt-killed", status: "terminated" },
{ attemptId: "attempt-resumed", status: "succeeded" },
]);
- expect(final?.usageLedger).toHaveLength(2);
+ expect(final?.usageLedger).toHaveLength(3);
expect(final && reconcileCapabilityLiveUsage(final)).toEqual({
- providerCalls: 2,
- inputTokens: 180,
+ providerCalls: 3,
+ providerRequests: 0,
+ inputTokens: 280,
outputTokens: 30,
cachedInputTokens: 40,
reasoningTokens: 5,
@@ -712,7 +721,7 @@ describe("Capability live runnerd and Codex session", () => {
{ attemptId: "attempt-real-killed", status: "terminated" },
{ attemptId: "attempt-real-resumed", status: "succeeded", resumeOf: "attempt-real-killed" },
]);
- expect(final?.usageLedger).toHaveLength(1);
+ expect(final?.usageLedger).toHaveLength(2);
expect(final?.terminalTurns).toEqual(expect.arrayContaining([
expect.objectContaining({ turnId: "turn-1", status: "interrupted" }),
expect.objectContaining({ turnId: "turn-2", status: "completed" }),
diff --git a/packages/paperclip-runner/src/live/live-session.ts b/packages/paperclip-runner/src/live/live-session.ts
index 48de2bc6eff..6207c009731 100644
--- a/packages/paperclip-runner/src/live/live-session.ts
+++ b/packages/paperclip-runner/src/live/live-session.ts
@@ -121,6 +121,7 @@ export interface CapabilityLiveUsageReceipt {
providerResponseId: string | null;
turnId: string | null;
providerCalls: number;
+ providerRequests: number;
inputTokens: number;
outputTokens: number;
cachedInputTokens: number;
@@ -142,6 +143,7 @@ export interface RecordCapabilityLiveUsageInput {
providerResponseId?: string | null;
turnId?: string | null;
providerCalls?: number;
+ providerRequests?: number;
inputTokens?: number;
outputTokens?: number;
cachedInputTokens?: number;
@@ -462,6 +464,7 @@ export function assertCapabilityLiveSessionSnapshot(
}
for (const name of [
"providerCalls",
+ "providerRequests",
"inputTokens",
"outputTokens",
"cachedInputTokens",
@@ -506,6 +509,7 @@ export function reconcileCapabilityLiveUsage(
const unique = new Map((snapshot.usageLedger ?? []).map((receipt) => [receipt.receiptId, receipt]));
return [...unique.values()].reduce((total, receipt) => ({
providerCalls: total.providerCalls + receipt.providerCalls,
+ providerRequests: total.providerRequests + receipt.providerRequests,
inputTokens: total.inputTokens + receipt.inputTokens,
outputTokens: total.outputTokens + receipt.outputTokens,
cachedInputTokens: total.cachedInputTokens + receipt.cachedInputTokens,
@@ -513,6 +517,7 @@ export function reconcileCapabilityLiveUsage(
costNanodollars: total.costNanodollars + receipt.costNanodollars,
}), {
providerCalls: 0,
+ providerRequests: 0,
inputTokens: 0,
outputTokens: 0,
cachedInputTokens: 0,
@@ -804,6 +809,7 @@ export class CapabilityLiveSession {
readonly #listeners = new Set();
#eventSeq = 0;
#turnEventCount = 0;
+ readonly #rawUsageByTurn = new Map();
private constructor(options: OpenSessionOptions & { snapshot?: CapabilityLiveSessionSnapshot }) {
this.#port = options.port;
@@ -1020,6 +1026,7 @@ export class CapabilityLiveSession {
providerResponseId: input.providerResponseId ?? null,
turnId: input.turnId ?? this.#activeTurnId,
providerCalls: nonNegativeInteger(input.providerCalls ?? 1, "provider_calls"),
+ providerRequests: nonNegativeInteger(input.providerRequests, "provider_requests"),
inputTokens: nonNegativeInteger(input.inputTokens, "input_tokens"),
outputTokens: nonNegativeInteger(input.outputTokens, "output_tokens"),
cachedInputTokens: nonNegativeInteger(input.cachedInputTokens, "cached_input_tokens"),
@@ -1112,10 +1119,51 @@ export class CapabilityLiveSession {
if (draft !== undefined && draft.turnId === null) draft.turnId = turnId;
await this.#persist();
const result = await terminal;
+ await this.#captureTurnUsage(result.turnId);
await this.#persist();
return { ...result, snapshot: this.snapshot() };
}
+ async #captureTurnUsage(turnId: string): Promise {
+ if (this.#transport === null) throw new Error("capability_live_usage_transport_missing");
+ const captured = this.#rawUsageByTurn.get(turnId);
+ const read = captured === undefined ? await this.#transport.request("thread/read", {
+ threadId: this.#providerThreadId,
+ includeTurns: true,
+ }) : {};
+ const thread = record(read.thread);
+ const usage = record(thread.tokenUsage ?? read.tokenUsage);
+ const total = record(captured ?? usage.total ?? usage.totalTokenUsage ?? usage.total_token_usage);
+ const integer = (...names: string[]): number => {
+ for (const name of names) {
+ const value = total[name];
+ if (Number.isSafeInteger(value) && Number(value) >= 0) return Number(value);
+ }
+ return 0;
+ };
+ const cumulative = {
+ inputTokens: integer("inputTokens", "input_tokens"),
+ outputTokens: integer("outputTokens", "output_tokens"),
+ cachedInputTokens: integer("cachedInputTokens", "cached_input_tokens"),
+ reasoningTokens: integer("reasoningOutputTokens", "reasoningTokens", "reasoning_output_tokens"),
+ };
+ if (cumulative.inputTokens + cumulative.outputTokens === 0) {
+ throw new Error("capability_live_usage_missing");
+ }
+ const prior = reconcileCapabilityLiveUsage(this.snapshot());
+ await this.recordUsage({
+ receiptId: `${turnId}:usage`,
+ turnId,
+ providerCalls: 1,
+ providerRequests: captured?.providerRequests ?? 0,
+ inputTokens: Math.max(0, cumulative.inputTokens - prior.inputTokens),
+ outputTokens: Math.max(0, cumulative.outputTokens - prior.outputTokens),
+ cachedInputTokens: Math.max(0, cumulative.cachedInputTokens - prior.cachedInputTokens),
+ reasoningTokens: Math.max(0, cumulative.reasoningTokens - prior.reasoningTokens),
+ costNanodollars: 0,
+ });
+ }
+
/**
* Runs an operator-authored DevTools call through the exact dispatcher used
* by Codex. It receives a real turn id, durable call/result evidence, and a
@@ -1380,7 +1428,7 @@ export class CapabilityLiveSession {
approvalPolicy: "never",
baseInstructions: LIVE_BASE_INSTRUCTIONS,
dynamicTools: tools.map(dynamicToolSpec),
- experimentalRawEvents: false,
+ experimentalRawEvents: true,
persistExtendedHistory: true,
});
const thread = record(opened.thread);
@@ -1565,7 +1613,20 @@ export class CapabilityLiveSession {
method: notification.method,
params: jsonValue(params),
});
- if (notification.method === "turn/started") {
+ if (notification.method === "rawResponse/completed") {
+ const rawUsage = record(params.usage);
+ if (turnId.length > 0 && Object.keys(rawUsage).length > 0) {
+ const previous = this.#rawUsageByTurn.get(turnId) ?? { providerRequests: 0, inputTokens: 0, outputTokens: 0, cachedInputTokens: 0, reasoningTokens: 0 };
+ const value = (name: string): number => Number.isSafeInteger(rawUsage[name]) && Number(rawUsage[name]) >= 0 ? Number(rawUsage[name]) : 0;
+ this.#rawUsageByTurn.set(turnId, {
+ providerRequests: previous.providerRequests + 1,
+ inputTokens: previous.inputTokens + value("inputTokens"),
+ outputTokens: previous.outputTokens + value("outputTokens"),
+ cachedInputTokens: previous.cachedInputTokens + value("cachedInputTokens"),
+ reasoningTokens: previous.reasoningTokens + value("reasoningOutputTokens"),
+ });
+ }
+ } else if (notification.method === "turn/started") {
if (turnId.length > 0) this.#activeTurnId = turnId;
this.#emit({ turnId: turnId || this.#activeTurnId, kind: "activity", reason: "turn_started" });
} else if (notification.method === "item/agentMessage/delta") {
diff --git a/packages/paperclip-runner/test/fixtures/fake-durable-codex-app-server.mjs b/packages/paperclip-runner/test/fixtures/fake-durable-codex-app-server.mjs
index f343e40ad16..0e744f37cab 100644
--- a/packages/paperclip-runner/test/fixtures/fake-durable-codex-app-server.mjs
+++ b/packages/paperclip-runner/test/fixtures/fake-durable-codex-app-server.mjs
@@ -74,6 +74,14 @@ function handleRequest(message) {
id: state.threadId,
sessionId: state.sessionId,
turns: Object.entries(state.turns).map(([turnId, status]) => ({ id: turnId, status })),
+ tokenUsage: {
+ total: {
+ inputTokens: state.nextTurn * 10,
+ cachedInputTokens: 0,
+ outputTokens: state.nextTurn * 2,
+ reasoningOutputTokens: 0,
+ },
+ },
},
},
});