fix(runner): report native Codex usage truthfully

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-08-26 21:37:42 +00:00
1 parent 27812e76ab
commit f73a5b474f
8 files changed
+209 -20

No files matched your search

+6 -1
View File
@@ -11,7 +11,12 @@ PRP transport. The transport authenticates and encrypts loopback WebSocket
sessions, persists an ACK-driven outbox and command journal, and reconnects with
a short-lived lease. The Rust runner now includes a Codex-only app-server
provider bridge with durable thread resume, cancellation, structured questions,
and provider-neutral event normalization. The root surface now also exposes an
and provider-neutral event normalization. Token-bearing Codex usage receipts
record at least one provider request when app-server omits its request count,
mark that count as an inexact minimum, and carry the provider turn ID for
correlation. App-server does not expose per-request response IDs or per-turn
cost, so those values remain null/zero only with explicit unavailable and
`unpriced` provenance. The root surface now also exposes an
authenticated durable PRP authority for server-side use. It stores only
bootstrap and reconnect credential digests, validates immutable run identity on
every connection and event, and persists commands and cumulative event ACK
@@ -95,7 +95,7 @@
{
"path": "schemas/usage.schema.json",
"id": "https://paperclip.dev/schemas/prp/v1/usage.schema.json",
"sha256": "28fdbb3202095144649dddc438e6270c66d599c6f7e86838687f58d644726456"
"sha256": "0855f8314ef0204eda03b3ebe1784a56d498e7dceba7b663a75ed095069d326d"
},
{
"path": "schemas/workspace-diff.schema.json",
@@ -8,7 +8,9 @@
"provider": { "type": "string", "minLength": 1, "maxLength": 80 },
"model": { "type": ["string", "null"], "maxLength": 240 },
"providerSessionId": { "type": ["string", "null"], "maxLength": 240 },
"providerTurnId": { "type": ["string", "null"], "maxLength": 240 },
"providerRequestId": { "type": ["string", "null"], "maxLength": 240 },
"providerRequestIdUnavailableReason": { "type": ["string", "null"], "maxLength": 240 },
"cumulative": { "$ref": "#/$defs/measurement" },
"runDelta": { "$ref": "#/$defs/measurement" }
},
@@ -24,7 +26,11 @@
"cacheWriteTokens": { "type": "integer", "minimum": 0 },
"activeSeconds": { "type": "number", "minimum": 0 },
"requests": { "type": "integer", "minimum": 0 },
"providerCostUsd": { "type": "number", "minimum": 0 }
"requestCountSource": { "enum": ["provider_reported", "token_bearing_turn_minimum", "unavailable"] },
"requestCountExact": { "type": "boolean" },
"providerCostUsd": { "type": "number", "minimum": 0 },
"providerCostStatus": { "enum": ["reported", "unpriced"] },
"providerCostUnavailableReason": { "type": ["string", "null"], "maxLength": 240 }
},
"additionalProperties": false
}
@@ -539,7 +539,17 @@ impl CodexProvider {
}
if let Some(method) = message.get("method").and_then(Value::as_str) {
let params = message.get("params").cloned().unwrap_or(Value::Null);
let mut params = message.get("params").cloned().unwrap_or(Value::Null);
if method == "thread/tokenUsage/updated" {
if let (Some(object), Some(turn_id)) = (
params.as_object_mut(),
self.active_provider_turn_id.as_deref(),
) {
object
.entry("turnId".to_owned())
.or_insert_with(|| json!(turn_id));
}
}
validate_notification_binding(
&self.thread_id,
self.active_provider_turn_id.as_deref(),
@@ -69,14 +69,51 @@ fn bounded_output(value: &str) -> Value {
}
fn measurement(value: &Value) -> Value {
let input_tokens = value
.get("inputTokens")
.and_then(Value::as_u64)
.unwrap_or(0);
let output_tokens = value
.get("outputTokens")
.and_then(Value::as_u64)
.unwrap_or(0);
let cache_read_tokens = value
.get("cachedInputTokens")
.or_else(|| value.get("cacheReadTokens"))
.and_then(Value::as_u64)
.unwrap_or(0);
let cache_write_tokens = value
.get("cacheWriteTokens")
.and_then(Value::as_u64)
.unwrap_or(0);
let reported_requests = value.get("requests").and_then(Value::as_u64);
let has_token_usage =
input_tokens > 0 || output_tokens > 0 || cache_read_tokens > 0 || cache_write_tokens > 0;
let (requests, request_count_source, request_count_exact) = match reported_requests {
Some(requests) => (requests, "provider_reported", true),
None if has_token_usage => (1, "token_bearing_turn_minimum", false),
None => (0, "unavailable", false),
};
let reported_cost = value
.get("providerCostUsd")
.and_then(Value::as_f64)
.filter(|value| *value >= 0.0);
json!({
"inputTokens": value.get("inputTokens").and_then(Value::as_u64).unwrap_or(0),
"outputTokens": value.get("outputTokens").and_then(Value::as_u64).unwrap_or(0),
"cacheReadTokens": value.get("cachedInputTokens").or_else(|| value.get("cacheReadTokens")).and_then(Value::as_u64).unwrap_or(0),
"cacheWriteTokens": value.get("cacheWriteTokens").and_then(Value::as_u64).unwrap_or(0),
"inputTokens": input_tokens,
"outputTokens": output_tokens,
"cacheReadTokens": cache_read_tokens,
"cacheWriteTokens": cache_write_tokens,
"activeSeconds": value.get("activeSeconds").and_then(Value::as_f64).filter(|value| *value >= 0.0).unwrap_or(0.0),
"requests": value.get("requests").and_then(Value::as_u64).unwrap_or(0),
"providerCostUsd": value.get("providerCostUsd").and_then(Value::as_f64).filter(|value| *value >= 0.0).unwrap_or(0.0),
"requests": requests,
"requestCountSource": request_count_source,
"requestCountExact": request_count_exact,
"providerCostUsd": reported_cost.unwrap_or(0.0),
"providerCostStatus": if reported_cost.is_some() { "reported" } else { "unpriced" },
"providerCostUnavailableReason": if reported_cost.is_some() {
Value::Null
} else {
json!("codex_app_server_does_not_report_per_turn_cost")
},
})
}
@@ -201,6 +238,17 @@ pub fn normalize_codex_notification(method: &str, params: &Value) -> Vec<Normali
.and_then(|value| value.get("last"))
.or_else(|| params.get("last"))
.unwrap_or(cumulative);
let provider_request_id = params
.get("responseId")
.or_else(|| params.get("requestId"))
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.map(|value| bounded_text(value, 240));
let provider_request_id_unavailable_reason = if provider_request_id.is_some() {
Value::Null
} else {
json!("codex_app_server_does_not_expose_per_request_id")
};
push(
&mut events,
"usage.reported",
@@ -209,7 +257,9 @@ pub fn normalize_codex_notification(method: &str, params: &Value) -> Vec<Normali
"provider": "codex",
"model": params.get("model").and_then(Value::as_str).map(|value| bounded_text(value, 240)),
"providerSessionId": params.get("threadId").and_then(Value::as_str).map(|value| bounded_text(value, 240)),
"providerRequestId": Value::Null,
"providerTurnId": params.get("turnId").and_then(Value::as_str).map(|value| bounded_text(value, 240)),
"providerRequestId": provider_request_id,
"providerRequestIdUnavailableReason": provider_request_id_unavailable_reason,
"cumulative": measurement(cumulative),
"runDelta": measurement(run_delta),
}),
@@ -370,6 +420,54 @@ mod tests {
);
assert_eq!(usage[0].event_type, "usage.reported");
assert_eq!(usage[0].payload["cumulative"]["inputTokens"], 12);
assert_eq!(usage[0].payload["cumulative"]["requests"], 1);
assert_eq!(
usage[0].payload["cumulative"]["requestCountSource"],
"token_bearing_turn_minimum"
);
assert_eq!(usage[0].payload["cumulative"]["requestCountExact"], false);
assert_eq!(
usage[0].payload["cumulative"]["providerCostStatus"],
"unpriced"
);
assert_eq!(
usage[0].payload["providerRequestIdUnavailableReason"],
"codex_app_server_does_not_expose_per_request_id"
);
assert_eq!(usage[0].priority, EventPriority::P0);
}
#[test]
fn preserves_provider_reported_request_and_cost_receipts() {
let usage = normalize_codex_notification(
"thread/tokenUsage/updated",
&json!({
"responseId": "resp_123",
"tokenUsage": {
"total": {"inputTokens": 12, "outputTokens": 3, "requests": 2, "providerCostUsd": 0.04},
"last": {"inputTokens": 5, "outputTokens": 1, "requests": 1, "providerCostUsd": 0.01}
}
}),
);
assert_eq!(usage[0].payload["providerRequestId"], "resp_123");
assert_eq!(
usage[0].payload["providerRequestIdUnavailableReason"],
Value::Null
);
assert_eq!(usage[0].payload["runDelta"]["requests"], 1);
assert_eq!(
usage[0].payload["runDelta"]["requestCountSource"],
"provider_reported"
);
assert_eq!(usage[0].payload["runDelta"]["requestCountExact"], true);
assert_eq!(usage[0].payload["runDelta"]["providerCostUsd"], 0.01);
assert_eq!(
usage[0].payload["runDelta"]["providerCostStatus"],
"reported"
);
assert_eq!(
usage[0].payload["runDelta"]["providerCostUnavailableReason"],
Value::Null
);
}
}
@@ -116,15 +116,18 @@ fn codex_transport_buffers_notifications_while_waiting_for_responses() {
.start_turn("Complete the fake task.", &config.cwd)
.expect("start provider turn");
let mut event_types = Vec::new();
let mut usage_provider_turn_id = None;
for _ in 0..16 {
if let Some(CodexProviderEvent::Notification { method, params }) =
provider.poll().expect("poll provider event")
{
event_types.extend(
normalize_codex_notification(&method, &params)
.into_iter()
.map(|event| event.event_type),
);
for event in normalize_codex_notification(&method, &params) {
if event.event_type == "usage.reported" {
usage_provider_turn_id =
event.payload["providerTurnId"].as_str().map(str::to_owned);
}
event_types.push(event.event_type);
}
}
if event_types.iter().any(|event| event == "turn.completed") {
break;
@@ -134,6 +137,7 @@ fn codex_transport_buffers_notifications_while_waiting_for_responses() {
assert!(event_types.iter().any(|event| event == "item.completed"));
assert!(event_types.iter().any(|event| event == "usage.reported"));
assert!(event_types.iter().any(|event| event == "turn.completed"));
assert_eq!(usage_provider_turn_id.as_deref(), Some("provider-turn-1"));
provider.shutdown().expect("stop provider");
fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
}
@@ -1883,6 +1883,13 @@ export const usageSchema = {
],
"maxLength": 240
},
"providerTurnId": {
"type": [
"string",
"null"
],
"maxLength": 240
},
"providerRequestId": {
"type": [
"string",
@@ -1890,6 +1897,13 @@ export const usageSchema = {
],
"maxLength": 240
},
"providerRequestIdUnavailableReason": {
"type": [
"string",
"null"
],
"maxLength": 240
},
"cumulative": {
"$ref": "#/$defs/measurement"
},
@@ -1935,9 +1949,32 @@ export const usageSchema = {
"type": "integer",
"minimum": 0
},
"requestCountSource": {
"enum": [
"provider_reported",
"token_bearing_turn_minimum",
"unavailable"
]
},
"requestCountExact": {
"type": "boolean"
},
"providerCostUsd": {
"type": "number",
"minimum": 0
},
"providerCostStatus": {
"enum": [
"reported",
"unpriced"
]
},
"providerCostUnavailableReason": {
"type": [
"string",
"null"
],
"maxLength": 240
}
},
"additionalProperties": false
@@ -333,13 +333,28 @@ describeEmbeddedPostgres("native Codex server vertical slice", () => {
{ body: "Native resume completed one semantic effect." },
]);
const usageReports = nativeEvents
.filter((event) => event.eventType === "usage.reported")
.map((event) => event.payload?.prpEvent)
.filter((event): event is Record<string, unknown> => Boolean(event));
const finalUsagePayload = usageReports.at(-1)?.payload as
| Record<string, unknown>
| undefined;
const finalCumulative = finalUsagePayload?.cumulative as
| Record<string, unknown>
| undefined;
expect(finalUsagePayload?.providerTurnId).toEqual(expect.any(String));
expect(finalCumulative?.requests).toBeGreaterThanOrEqual(1);
expect(finalCumulative?.providerCostStatus).toBe("unpriced");
expect(finalCumulative?.providerCostUnavailableReason).toBe(
"codex_app_server_does_not_report_per_turn_cost",
);
if (liveCodex) {
const usageReports = nativeEvents
.filter((event) => event.eventType === "usage.reported")
.map((event) => event.payload?.prpEvent)
.filter((event): event is Record<string, unknown> => Boolean(event));
console.log("NATIVE_CODEX_RESUME_PROOF", JSON.stringify({
schema: "paperclip-runner/native-resume-live-gate/v1",
appCommit: process.env.PAPERCLIP_LIVE_CODEX_APP_COMMIT ?? null,
model: "gpt-5.6-sol",
runId,
providerSessionId: result.sessionParams?.sessionId ?? null,
semanticCallId,
@@ -353,6 +368,20 @@ describeEmbeddedPostgres("native Codex server vertical slice", () => {
(event) => event.eventType === "semantic_tool.result",
).length,
controlPlaneEffectCount: progressEffects.length,
accounting: {
providerCalls: 1,
providerRequests: finalCumulative?.requests ?? null,
requestCountSource: finalCumulative?.requestCountSource ?? null,
requestCountExact: finalCumulative?.requestCountExact ?? null,
providerTurnId: finalUsagePayload?.providerTurnId ?? null,
providerRequestId: finalUsagePayload?.providerRequestId ?? null,
providerRequestIdUnavailableReason:
finalUsagePayload?.providerRequestIdUnavailableReason ?? null,
providerCostUsd: finalCumulative?.providerCostUsd ?? null,
providerCostStatus: finalCumulative?.providerCostStatus ?? null,
providerCostUnavailableReason:
finalCumulative?.providerCostUnavailableReason ?? null,
},
usageReports,
}));
return;