mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-25 17:45:53 +01:00
fix(runner): report native Codex usage truthfully
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
27812e76ab
commit
f73a5b474f
8 files changed
+209
-20
No files matched your search
@@ -11,7 +11,12 @@ PRP transport. The transport authenticates and encrypts loopback WebSocket
|
||||
sessions, persists an ACK-driven outbox and command journal, and reconnects with
|
||||
a short-lived lease. The Rust runner now includes a Codex-only app-server
|
||||
provider bridge with durable thread resume, cancellation, structured questions,
|
||||
and provider-neutral event normalization. The root surface now also exposes an
|
||||
and provider-neutral event normalization. Token-bearing Codex usage receipts
|
||||
record at least one provider request when app-server omits its request count,
|
||||
mark that count as an inexact minimum, and carry the provider turn ID for
|
||||
correlation. App-server does not expose per-request response IDs or per-turn
|
||||
cost, so those values remain null/zero only with explicit unavailable and
|
||||
`unpriced` provenance. The root surface now also exposes an
|
||||
authenticated durable PRP authority for server-side use. It stores only
|
||||
bootstrap and reconnect credential digests, validates immutable run identity on
|
||||
every connection and event, and persists commands and cumulative event ACK
|
||||
|
||||
@@ -95,7 +95,7 @@
|
||||
{
|
||||
"path": "schemas/usage.schema.json",
|
||||
"id": "https://paperclip.dev/schemas/prp/v1/usage.schema.json",
|
||||
"sha256": "28fdbb3202095144649dddc438e6270c66d599c6f7e86838687f58d644726456"
|
||||
"sha256": "0855f8314ef0204eda03b3ebe1784a56d498e7dceba7b663a75ed095069d326d"
|
||||
},
|
||||
{
|
||||
"path": "schemas/workspace-diff.schema.json",
|
||||
|
||||
@@ -8,7 +8,9 @@
|
||||
"provider": { "type": "string", "minLength": 1, "maxLength": 80 },
|
||||
"model": { "type": ["string", "null"], "maxLength": 240 },
|
||||
"providerSessionId": { "type": ["string", "null"], "maxLength": 240 },
|
||||
"providerTurnId": { "type": ["string", "null"], "maxLength": 240 },
|
||||
"providerRequestId": { "type": ["string", "null"], "maxLength": 240 },
|
||||
"providerRequestIdUnavailableReason": { "type": ["string", "null"], "maxLength": 240 },
|
||||
"cumulative": { "$ref": "#/$defs/measurement" },
|
||||
"runDelta": { "$ref": "#/$defs/measurement" }
|
||||
},
|
||||
@@ -24,7 +26,11 @@
|
||||
"cacheWriteTokens": { "type": "integer", "minimum": 0 },
|
||||
"activeSeconds": { "type": "number", "minimum": 0 },
|
||||
"requests": { "type": "integer", "minimum": 0 },
|
||||
"providerCostUsd": { "type": "number", "minimum": 0 }
|
||||
"requestCountSource": { "enum": ["provider_reported", "token_bearing_turn_minimum", "unavailable"] },
|
||||
"requestCountExact": { "type": "boolean" },
|
||||
"providerCostUsd": { "type": "number", "minimum": 0 },
|
||||
"providerCostStatus": { "enum": ["reported", "unpriced"] },
|
||||
"providerCostUnavailableReason": { "type": ["string", "null"], "maxLength": 240 }
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
|
||||
@@ -539,7 +539,17 @@ impl CodexProvider {
|
||||
}
|
||||
|
||||
if let Some(method) = message.get("method").and_then(Value::as_str) {
|
||||
let params = message.get("params").cloned().unwrap_or(Value::Null);
|
||||
let mut params = message.get("params").cloned().unwrap_or(Value::Null);
|
||||
if method == "thread/tokenUsage/updated" {
|
||||
if let (Some(object), Some(turn_id)) = (
|
||||
params.as_object_mut(),
|
||||
self.active_provider_turn_id.as_deref(),
|
||||
) {
|
||||
object
|
||||
.entry("turnId".to_owned())
|
||||
.or_insert_with(|| json!(turn_id));
|
||||
}
|
||||
}
|
||||
validate_notification_binding(
|
||||
&self.thread_id,
|
||||
self.active_provider_turn_id.as_deref(),
|
||||
|
||||
@@ -69,14 +69,51 @@ fn bounded_output(value: &str) -> Value {
|
||||
}
|
||||
|
||||
fn measurement(value: &Value) -> Value {
|
||||
let input_tokens = value
|
||||
.get("inputTokens")
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or(0);
|
||||
let output_tokens = value
|
||||
.get("outputTokens")
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or(0);
|
||||
let cache_read_tokens = value
|
||||
.get("cachedInputTokens")
|
||||
.or_else(|| value.get("cacheReadTokens"))
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or(0);
|
||||
let cache_write_tokens = value
|
||||
.get("cacheWriteTokens")
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or(0);
|
||||
let reported_requests = value.get("requests").and_then(Value::as_u64);
|
||||
let has_token_usage =
|
||||
input_tokens > 0 || output_tokens > 0 || cache_read_tokens > 0 || cache_write_tokens > 0;
|
||||
let (requests, request_count_source, request_count_exact) = match reported_requests {
|
||||
Some(requests) => (requests, "provider_reported", true),
|
||||
None if has_token_usage => (1, "token_bearing_turn_minimum", false),
|
||||
None => (0, "unavailable", false),
|
||||
};
|
||||
let reported_cost = value
|
||||
.get("providerCostUsd")
|
||||
.and_then(Value::as_f64)
|
||||
.filter(|value| *value >= 0.0);
|
||||
json!({
|
||||
"inputTokens": value.get("inputTokens").and_then(Value::as_u64).unwrap_or(0),
|
||||
"outputTokens": value.get("outputTokens").and_then(Value::as_u64).unwrap_or(0),
|
||||
"cacheReadTokens": value.get("cachedInputTokens").or_else(|| value.get("cacheReadTokens")).and_then(Value::as_u64).unwrap_or(0),
|
||||
"cacheWriteTokens": value.get("cacheWriteTokens").and_then(Value::as_u64).unwrap_or(0),
|
||||
"inputTokens": input_tokens,
|
||||
"outputTokens": output_tokens,
|
||||
"cacheReadTokens": cache_read_tokens,
|
||||
"cacheWriteTokens": cache_write_tokens,
|
||||
"activeSeconds": value.get("activeSeconds").and_then(Value::as_f64).filter(|value| *value >= 0.0).unwrap_or(0.0),
|
||||
"requests": value.get("requests").and_then(Value::as_u64).unwrap_or(0),
|
||||
"providerCostUsd": value.get("providerCostUsd").and_then(Value::as_f64).filter(|value| *value >= 0.0).unwrap_or(0.0),
|
||||
"requests": requests,
|
||||
"requestCountSource": request_count_source,
|
||||
"requestCountExact": request_count_exact,
|
||||
"providerCostUsd": reported_cost.unwrap_or(0.0),
|
||||
"providerCostStatus": if reported_cost.is_some() { "reported" } else { "unpriced" },
|
||||
"providerCostUnavailableReason": if reported_cost.is_some() {
|
||||
Value::Null
|
||||
} else {
|
||||
json!("codex_app_server_does_not_report_per_turn_cost")
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -201,6 +238,17 @@ pub fn normalize_codex_notification(method: &str, params: &Value) -> Vec<Normali
|
||||
.and_then(|value| value.get("last"))
|
||||
.or_else(|| params.get("last"))
|
||||
.unwrap_or(cumulative);
|
||||
let provider_request_id = params
|
||||
.get("responseId")
|
||||
.or_else(|| params.get("requestId"))
|
||||
.and_then(Value::as_str)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(|value| bounded_text(value, 240));
|
||||
let provider_request_id_unavailable_reason = if provider_request_id.is_some() {
|
||||
Value::Null
|
||||
} else {
|
||||
json!("codex_app_server_does_not_expose_per_request_id")
|
||||
};
|
||||
push(
|
||||
&mut events,
|
||||
"usage.reported",
|
||||
@@ -209,7 +257,9 @@ pub fn normalize_codex_notification(method: &str, params: &Value) -> Vec<Normali
|
||||
"provider": "codex",
|
||||
"model": params.get("model").and_then(Value::as_str).map(|value| bounded_text(value, 240)),
|
||||
"providerSessionId": params.get("threadId").and_then(Value::as_str).map(|value| bounded_text(value, 240)),
|
||||
"providerRequestId": Value::Null,
|
||||
"providerTurnId": params.get("turnId").and_then(Value::as_str).map(|value| bounded_text(value, 240)),
|
||||
"providerRequestId": provider_request_id,
|
||||
"providerRequestIdUnavailableReason": provider_request_id_unavailable_reason,
|
||||
"cumulative": measurement(cumulative),
|
||||
"runDelta": measurement(run_delta),
|
||||
}),
|
||||
@@ -370,6 +420,54 @@ mod tests {
|
||||
);
|
||||
assert_eq!(usage[0].event_type, "usage.reported");
|
||||
assert_eq!(usage[0].payload["cumulative"]["inputTokens"], 12);
|
||||
assert_eq!(usage[0].payload["cumulative"]["requests"], 1);
|
||||
assert_eq!(
|
||||
usage[0].payload["cumulative"]["requestCountSource"],
|
||||
"token_bearing_turn_minimum"
|
||||
);
|
||||
assert_eq!(usage[0].payload["cumulative"]["requestCountExact"], false);
|
||||
assert_eq!(
|
||||
usage[0].payload["cumulative"]["providerCostStatus"],
|
||||
"unpriced"
|
||||
);
|
||||
assert_eq!(
|
||||
usage[0].payload["providerRequestIdUnavailableReason"],
|
||||
"codex_app_server_does_not_expose_per_request_id"
|
||||
);
|
||||
assert_eq!(usage[0].priority, EventPriority::P0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_provider_reported_request_and_cost_receipts() {
|
||||
let usage = normalize_codex_notification(
|
||||
"thread/tokenUsage/updated",
|
||||
&json!({
|
||||
"responseId": "resp_123",
|
||||
"tokenUsage": {
|
||||
"total": {"inputTokens": 12, "outputTokens": 3, "requests": 2, "providerCostUsd": 0.04},
|
||||
"last": {"inputTokens": 5, "outputTokens": 1, "requests": 1, "providerCostUsd": 0.01}
|
||||
}
|
||||
}),
|
||||
);
|
||||
assert_eq!(usage[0].payload["providerRequestId"], "resp_123");
|
||||
assert_eq!(
|
||||
usage[0].payload["providerRequestIdUnavailableReason"],
|
||||
Value::Null
|
||||
);
|
||||
assert_eq!(usage[0].payload["runDelta"]["requests"], 1);
|
||||
assert_eq!(
|
||||
usage[0].payload["runDelta"]["requestCountSource"],
|
||||
"provider_reported"
|
||||
);
|
||||
assert_eq!(usage[0].payload["runDelta"]["requestCountExact"], true);
|
||||
assert_eq!(usage[0].payload["runDelta"]["providerCostUsd"], 0.01);
|
||||
assert_eq!(
|
||||
usage[0].payload["runDelta"]["providerCostStatus"],
|
||||
"reported"
|
||||
);
|
||||
assert_eq!(
|
||||
usage[0].payload["runDelta"]["providerCostUnavailableReason"],
|
||||
Value::Null
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -116,15 +116,18 @@ fn codex_transport_buffers_notifications_while_waiting_for_responses() {
|
||||
.start_turn("Complete the fake task.", &config.cwd)
|
||||
.expect("start provider turn");
|
||||
let mut event_types = Vec::new();
|
||||
let mut usage_provider_turn_id = None;
|
||||
for _ in 0..16 {
|
||||
if let Some(CodexProviderEvent::Notification { method, params }) =
|
||||
provider.poll().expect("poll provider event")
|
||||
{
|
||||
event_types.extend(
|
||||
normalize_codex_notification(&method, ¶ms)
|
||||
.into_iter()
|
||||
.map(|event| event.event_type),
|
||||
);
|
||||
for event in normalize_codex_notification(&method, ¶ms) {
|
||||
if event.event_type == "usage.reported" {
|
||||
usage_provider_turn_id =
|
||||
event.payload["providerTurnId"].as_str().map(str::to_owned);
|
||||
}
|
||||
event_types.push(event.event_type);
|
||||
}
|
||||
}
|
||||
if event_types.iter().any(|event| event == "turn.completed") {
|
||||
break;
|
||||
@@ -134,6 +137,7 @@ fn codex_transport_buffers_notifications_while_waiting_for_responses() {
|
||||
assert!(event_types.iter().any(|event| event == "item.completed"));
|
||||
assert!(event_types.iter().any(|event| event == "usage.reported"));
|
||||
assert!(event_types.iter().any(|event| event == "turn.completed"));
|
||||
assert_eq!(usage_provider_turn_id.as_deref(), Some("provider-turn-1"));
|
||||
provider.shutdown().expect("stop provider");
|
||||
fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
|
||||
}
|
||||
|
||||
@@ -1883,6 +1883,13 @@ export const usageSchema = {
|
||||
],
|
||||
"maxLength": 240
|
||||
},
|
||||
"providerTurnId": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
],
|
||||
"maxLength": 240
|
||||
},
|
||||
"providerRequestId": {
|
||||
"type": [
|
||||
"string",
|
||||
@@ -1890,6 +1897,13 @@ export const usageSchema = {
|
||||
],
|
||||
"maxLength": 240
|
||||
},
|
||||
"providerRequestIdUnavailableReason": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
],
|
||||
"maxLength": 240
|
||||
},
|
||||
"cumulative": {
|
||||
"$ref": "#/$defs/measurement"
|
||||
},
|
||||
@@ -1935,9 +1949,32 @@ export const usageSchema = {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"requestCountSource": {
|
||||
"enum": [
|
||||
"provider_reported",
|
||||
"token_bearing_turn_minimum",
|
||||
"unavailable"
|
||||
]
|
||||
},
|
||||
"requestCountExact": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"providerCostUsd": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"providerCostStatus": {
|
||||
"enum": [
|
||||
"reported",
|
||||
"unpriced"
|
||||
]
|
||||
},
|
||||
"providerCostUnavailableReason": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
],
|
||||
"maxLength": 240
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
|
||||
@@ -333,13 +333,28 @@ describeEmbeddedPostgres("native Codex server vertical slice", () => {
|
||||
{ body: "Native resume completed one semantic effect." },
|
||||
]);
|
||||
|
||||
const usageReports = nativeEvents
|
||||
.filter((event) => event.eventType === "usage.reported")
|
||||
.map((event) => event.payload?.prpEvent)
|
||||
.filter((event): event is Record<string, unknown> => Boolean(event));
|
||||
const finalUsagePayload = usageReports.at(-1)?.payload as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
const finalCumulative = finalUsagePayload?.cumulative as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
expect(finalUsagePayload?.providerTurnId).toEqual(expect.any(String));
|
||||
expect(finalCumulative?.requests).toBeGreaterThanOrEqual(1);
|
||||
expect(finalCumulative?.providerCostStatus).toBe("unpriced");
|
||||
expect(finalCumulative?.providerCostUnavailableReason).toBe(
|
||||
"codex_app_server_does_not_report_per_turn_cost",
|
||||
);
|
||||
|
||||
if (liveCodex) {
|
||||
const usageReports = nativeEvents
|
||||
.filter((event) => event.eventType === "usage.reported")
|
||||
.map((event) => event.payload?.prpEvent)
|
||||
.filter((event): event is Record<string, unknown> => Boolean(event));
|
||||
console.log("NATIVE_CODEX_RESUME_PROOF", JSON.stringify({
|
||||
schema: "paperclip-runner/native-resume-live-gate/v1",
|
||||
appCommit: process.env.PAPERCLIP_LIVE_CODEX_APP_COMMIT ?? null,
|
||||
model: "gpt-5.6-sol",
|
||||
runId,
|
||||
providerSessionId: result.sessionParams?.sessionId ?? null,
|
||||
semanticCallId,
|
||||
@@ -353,6 +368,20 @@ describeEmbeddedPostgres("native Codex server vertical slice", () => {
|
||||
(event) => event.eventType === "semantic_tool.result",
|
||||
).length,
|
||||
controlPlaneEffectCount: progressEffects.length,
|
||||
accounting: {
|
||||
providerCalls: 1,
|
||||
providerRequests: finalCumulative?.requests ?? null,
|
||||
requestCountSource: finalCumulative?.requestCountSource ?? null,
|
||||
requestCountExact: finalCumulative?.requestCountExact ?? null,
|
||||
providerTurnId: finalUsagePayload?.providerTurnId ?? null,
|
||||
providerRequestId: finalUsagePayload?.providerRequestId ?? null,
|
||||
providerRequestIdUnavailableReason:
|
||||
finalUsagePayload?.providerRequestIdUnavailableReason ?? null,
|
||||
providerCostUsd: finalCumulative?.providerCostUsd ?? null,
|
||||
providerCostStatus: finalCumulative?.providerCostStatus ?? null,
|
||||
providerCostUnavailableReason:
|
||||
finalCumulative?.providerCostUnavailableReason ?? null,
|
||||
},
|
||||
usageReports,
|
||||
}));
|
||||
return;
|
||||
|
||||
Reference in new issue
Block a user