From 842925d08c4f34b29f3ccb0782dcf7e852362cb4 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 7 Oct 2026 23:24:21 -0500 Subject: [PATCH] fix(hermes): settle closed retry work with unknown usage Co-Authored-By: Paperclip --- doc/plans/2026-10-06-hermes-native-runner.md | 47 ++++++++ packages/adapter-utils/src/index.ts | 1 + packages/adapter-utils/src/types.ts | 9 ++ packages/paperclip-runner/docs/hermes.md | 10 +- .../scripts/qualify-hermes-runnerd.test.mjs | 3 +- .../src/drivers/acpx/hermes-distributions.ts | 4 +- .../src/providers/hermes/bridge.py | 8 ++ .../src/providers/hermes/test_bridge.py | 17 ++- server/src/services/heartbeat.ts | 8 +- .../native-session-executor.test.ts | 18 ++- .../native-runtime/native-session-executor.ts | 36 ++++-- server/src/services/usage-receipts.test.ts | 104 ++++++++++++++++++ server/src/services/usage-receipts.ts | 23 +++- .../runner-e2e/hermes-api-connections.test.ts | 2 +- tests/runner-e2e/hermes-api-connections.ts | 2 +- 15 files changed, 261 insertions(+), 31 deletions(-) create mode 100644 server/src/services/usage-receipts.test.ts diff --git a/doc/plans/2026-10-06-hermes-native-runner.md b/doc/plans/2026-10-06-hermes-native-runner.md index 9fa1245dab6..24f905aa084 100644 --- a/doc/plans/2026-10-06-hermes-native-runner.md +++ b/doc/plans/2026-10-06-hermes-native-runner.md @@ -907,3 +907,50 @@ Fresh Linux materialization, current-head CI/review, live reported-cost browser proof and broader original release gates remain required. Existing cloud images and prior results keep their old runtime/source identities. No local Docker, paid inference or additional reservation was used for these offline checks. + +### 2026-10-07 reported-cost live attempt and retry correction + +The clean `f4ae618d` Mac browser campaign +`hermes-macos-billing-openrouter-f4ae618d38-20261008` failed its settlement +oracle after 110.244 seconds; cleanup passed. The original failure and its +`cleanup_failure` classification remain unchanged. Its public run receipt +reports `$0.001160247`, 61,157 input and 271 output tokens, with completed +accounting and an idle agent. The public company response omits `pauseReason`; +the oracle incorrectly required it to be null. The corrected predicate uses +the authoritative company status and agent pause state. The final screenshot +shows Done, one final marker, a thought card and an available composer. +Continued use was not tested; this failed attempt is not a qualification pass. + +Review identified that a failed HTTP attempt can leave token totals unknown +after a successful SDK retry. The controller now separates final measurement +completeness from provider closure with the optional closed +`paperclip.accounting.settlement/v1` object. A known subtotal can settle once +as unpriced after native finalization. Unknown tokens stay null. Invalid +settlement, unfinished attempts, capture failure and unclosed coordinator state +cannot acknowledge debt. Legacy incomplete receipts retain their old behavior. + +Paperclip owns task titles, so the managed profile disables Hermes's paid +background title upgrade while retaining its immediate derived session title. +This avoids title inference after a turn receipt closes. The updated Mac +closure is `ad1e555296e51be6d20a7234daeb7028570c7ee8fee9d27fb88c4f550491a07f`; +the derived Linux closure is `353a942b2a88542db8537611535898ae6088045608b88b4ddab491a6b000b8cf`. +Both predecessor manifests were authenticated, and the task-owned Mac asset +was verified file by file before and after the bridge-only transformation. +The full key reservation remains held because the failed campaign does not +prove complete account or background inference spend. Fresh native, live, +Linux, CI and review proof remain required. No local Docker is used. + +The local full Vitest run at `f4ae618d` was interrupted after review required +a source change. Its log is retained and is not a passing full-suite result. +The earlier repository typecheck passed. Full checks must pass at the final +reviewed head before release qualification can complete. + +The correction passes all 615 native-executor and durable-receipt checks, +35 pinned Python checks, 19 public settlement-oracle checks, repository-wide +typecheck and the full build. The native ACPX restore/control fixture passes +in 77.361 seconds. The Rust/image/completion fixture passes in 20.061 seconds +after its inference-only assertion was corrected to exclude metadata probes. +The adapter failure-path suite passes all 31 tests after the build completes; +the earlier concurrent-build test failures remain in their original log. +This is offline and deterministic proof. A new clean-head live campaign and +fresh CI/review remain required. Docker is confirmed stopped. diff --git a/packages/adapter-utils/src/index.ts b/packages/adapter-utils/src/index.ts index 3240fdf4786..b409261198f 100644 --- a/packages/adapter-utils/src/index.ts +++ b/packages/adapter-utils/src/index.ts @@ -4,6 +4,7 @@ export type { AdapterRuntime, UsageSummary, AdapterUsageCheckpoint, + AdapterAccountingSettlement, AdapterBillingType, AdapterRuntimeServiceReport, AdapterExecutionResult, diff --git a/packages/adapter-utils/src/types.ts b/packages/adapter-utils/src/types.ts index d648b4fd3c1..df7d9466aa5 100644 --- a/packages/adapter-utils/src/types.ts +++ b/packages/adapter-utils/src/types.ts @@ -40,6 +40,13 @@ export interface UsageSummary { } /** Accounting-only snapshot. Never include prompt, response, or credentials. */ +export interface AdapterAccountingSettlement { + schema: "paperclip.accounting.settlement/v1"; + /** Owned provider work ended; unknown measurements remain explicitly unknown. */ + providerWorkEnded: true; + usageComplete: boolean; +} + export interface AdapterUsageCheckpoint { attemptId?: string; usage?: UsageSummary; @@ -57,6 +64,7 @@ export interface AdapterUsageCheckpoint { cacheAdjustedCostUsd?: number | null; providerRequestId?: string | null; complete: boolean; + settlement?: AdapterAccountingSettlement; } export type AdapterBillingType = @@ -143,6 +151,7 @@ export interface AdapterExecutionResult { pricingProvenance?: { source: "provider_reported" | "provider_invoice" | "operator" | "rate_card" | "unknown"; version?: string; evidence?: string; inputCentsPerMillion?: string; cachedInputCentsPerMillion?: string; cacheWriteCentsPerMillion?: string; outputCentsPerMillion?: string; serviceTier?: string; contextTier?: "short" | "long" }; providerRequestId?: string | null; usageComplete?: boolean; + settlement?: AdapterAccountingSettlement; /** * Provider-billed cost after prompt-cache discounts. Adapters should set * this when they expose it separately; otherwise the server treats a diff --git a/packages/paperclip-runner/docs/hermes.md b/packages/paperclip-runner/docs/hermes.md index b102df680ac..132f12b8b71 100644 --- a/packages/paperclip-runner/docs/hermes.md +++ b/packages/paperclip-runner/docs/hermes.md @@ -117,11 +117,19 @@ extend the task's execution deadline. using it for accounting. Cumulative session cost and native model-price estimates cannot supply this receipt. - Failed, interrupted, unsupported asynchronous and background delegated work - leave settlement incomplete. Known positive reported subtotals survive, but + leave measurement totals incomplete. Known positive reported subtotals survive, but missing charges remain unpriced. Explicit reported zero is accepted only for a complete receipt. Other provider/protocol routes retain their existing unavailable billed-cost behavior. Complete live billing, including delegated work, remains a qualification gate; these checks do not qualify a provider. +- Once the native provider boundary closes, an optional versioned accounting + settlement can acknowledge that known subtotal even when tokens or charges + are incomplete. Missing token totals stay unknown, incomplete charges stay + unpriced, and the existing controller finalization and capture-failure fences + still control acknowledgement. Older incomplete receipts remain pending. +- Paperclip owns task titles. Hermes keeps its immediate derived session title; + its paid background title upgrade is disabled in the managed profile so it + cannot start inference after a turn's usage receipt has closed. The managed execution middleware applies planning/read-only, protected-path and permission policy to inline, concurrent and delegated native tools. Assigned diff --git a/packages/paperclip-runner/scripts/qualify-hermes-runnerd.test.mjs b/packages/paperclip-runner/scripts/qualify-hermes-runnerd.test.mjs index 56d0e808f67..32cbec9110b 100644 --- a/packages/paperclip-runner/scripts/qualify-hermes-runnerd.test.mjs +++ b/packages/paperclip-runner/scripts/qualify-hermes-runnerd.test.mjs @@ -37,9 +37,9 @@ test('pinned Hermes streams images and semantic completion through Rust PRP and const server = createServer(async (req, res) => { const chunks = []; for await (const chunk of req) chunks.push(chunk); const body = JSON.parse(Buffer.concat(chunks).toString() || '{}'); - requests.push(body); assert.equal(req.headers.authorization, undefined); if (req.url !== '/v1/chat/completions') { res.writeHead(404).end(); return; } + requests.push(body); if (!body.stream) { res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify({ id: 'title', object: 'chat.completion', model: 'hermes-fixture', choices: [{ index: 0, message: { role: 'assistant', content: 'Fixture' }, finish_reason: 'stop' }] })); @@ -117,6 +117,7 @@ test('pinned Hermes streams images and semantic completion through Rust PRP and assert.ok(events.some(event => JSON.stringify(event.payload).includes('Checking the native runner path.')), 'Reasoning did not cross PRP'); assert.ok(events.some(event => JSON.stringify(event.payload).includes('Native Rust')), 'Text did not cross PRP'); assert.ok(requests.some(body => body.messages?.some(message => Array.isArray(message.content) && message.content.some(part => part.type === 'image_url' && part.image_url.url === `data:image/png;base64,${imageData}`))), 'Images did not cross TS/Rust/sidecar'); + assert.ok(requests.every(body => body.stream), 'Managed Hermes started auxiliary title inference'); const snapshot = await session.snapshot(); assert.equal(snapshot.semanticResult?.reportedWorkDisposition, 'done'); assert.equal(snapshot.terminal?.runTerminalState, 'succeeded'); diff --git a/packages/paperclip-runner/src/drivers/acpx/hermes-distributions.ts b/packages/paperclip-runner/src/drivers/acpx/hermes-distributions.ts index be8ab147214..8dfb88a4bef 100644 --- a/packages/paperclip-runner/src/drivers/acpx/hermes-distributions.ts +++ b/packages/paperclip-runner/src/drivers/acpx/hermes-distributions.ts @@ -1,5 +1,5 @@ /** Reviewed native closures. Setup cannot adopt a digest from downloaded files. */ export const HERMES_CLOSURES: Readonly> = Object.freeze({ - "darwin-arm64": "9f1af058a963305ccd5c4f1555f2828458f72677ceacea2de456ac503f7212de", - "linux-x64": "dd918ec15f5bd8025f3d01c4f2849eac4de8d59e237a520607f9c5f988a94645", + "darwin-arm64": "ad1e555296e51be6d20a7234daeb7028570c7ee8fee9d27fb88c4f550491a07f", + "linux-x64": "353a942b2a88542db8537611535898ae6088045608b88b4ddab491a6b000b8cf", }); diff --git a/packages/paperclip-runner/src/providers/hermes/bridge.py b/packages/paperclip-runner/src/providers/hermes/bridge.py index 8b14bf8b829..f278e090b91 100644 --- a/packages/paperclip-runner/src/providers/hermes/bridge.py +++ b/packages/paperclip-runner/src/providers/hermes/bridge.py @@ -534,9 +534,17 @@ def main(): asyncio.run(acp.run_agent(ManagedHermesACPAgent(), use_unstable_protocol=True)) +def disable_background_title_inference(): + # Paperclip owns task titles. Keep Hermes's immediate derived session title, + # but prevent a paid title-upgrade thread from outliving the turn receipt. + from agent import title_generator + title_generator._model_title_upgrade_enabled = lambda: False + + def configure_managed_runtime(): install_tool_process_policy() install_billing_capture() + disable_background_title_inference() from tools import delegate_tool_dispatch delegate_tool_dispatch._dispatch_background = fence_background_dispatch(delegate_tool_dispatch._dispatch_background) from agent import turn_response_check diff --git a/packages/paperclip-runner/src/providers/hermes/test_bridge.py b/packages/paperclip-runner/src/providers/hermes/test_bridge.py index b43ae9dcaea..0c1591348c0 100644 --- a/packages/paperclip-runner/src/providers/hermes/test_bridge.py +++ b/packages/paperclip-runner/src/providers/hermes/test_bridge.py @@ -6,12 +6,27 @@ from types import SimpleNamespace from unittest.mock import AsyncMock, Mock, patch from acp.schema import ClientCapabilities -from bridge import ManagedHermesACPAgent, ManagedSessionManager, ChannelClient, native_answers, question_set, install_no_auth_transport, turn_usage, USAGE_COUNTERS +from bridge import ManagedHermesACPAgent, ManagedSessionManager, ChannelClient, native_answers, question_set, install_no_auth_transport, disable_background_title_inference, turn_usage, USAGE_COUNTERS from policy import authorize_tool from billing import TurnBilling class Accounting(unittest.TestCase): + def test_managed_title_keeps_derived_title_without_background_inference(self): + from agent import title_generator, memory_provider + store = Mock() + store.get_session_title.return_value = None + store.set_auto_title.return_value = True + with patch.object(title_generator, "_model_title_upgrade_enabled"), \ + patch.object(title_generator, "_auto_title_enabled", return_value=True), \ + patch.object(memory_provider, "spawn_context_thread") as spawn: + disable_background_title_inference() + result = title_generator.maybe_auto_title(store, "session", "Investigate the runner retry accounting") + self.assertIsNone(result) + store.set_auto_title.assert_called_once_with("session", unittest.mock.ANY, source="derived") + self.assertTrue(store.set_auto_title.call_args.args[1]) + spawn.assert_not_called() + def test_usage_is_per_turn_and_cache_and_thought_are_not_double_counted(self): before = dict.fromkeys(USAGE_COUNTERS, 100) after = {key: value + 5 for key, value in before.items()} diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 20aac245750..10348479c59 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -25957,10 +25957,14 @@ export function heartbeatService( const cacheAdjustedCostUsd = adapterResult.costUsdExact != null && adapterResult.cacheAdjustedCostUsd == null ? null : resolveCacheAdjustedCostUsd(adapterResult); const usageJson: Record = { - accountingReceiptReady: adapterResult.usageComplete !== false, + accountingReceiptReady: adapterResult.usageComplete !== false || adapterResult.settlement?.providerWorkEnded === true, + accountingSettlement: adapterResult.settlement ?? null, + accountingUsageComplete: adapterResult.usageComplete !== false, costUsdExact: adapterResult.costUsdExact ?? null, providerRequestId: adapterResult.providerRequestId ?? null, - ...(normalizedUsage ?? {}), + ...(adapterResult.settlement?.usageComplete === false + ? { inputTokens: null, outputTokens: null, cachedInputTokens: null, cacheWriteTokens: null } + : normalizedUsage ?? {}), ...(adapterResult.usageByModel ? { usageByModel: adapterResult.usageByModel } : {}), ...(rawUsage ? { diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index a0ed214acec..d9633a01dd5 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -5555,17 +5555,19 @@ describe("native terminal-turn accounting", () => { if (scenario === "late_update") expect(result.usage).toMatchObject({ inputTokens: 32, outputTokens: 8 }); }); - it.each(["reported", "partial", "legacy", "wrong_account", "zero"] as const)("settles bound Hermes billing through the executor (%s)", async scenario => { + it.each(["reported", "partial", "partial_missing_tokens", "reported_missing_tokens", "legacy", "wrong_account", "zero"] as const)("settles bound Hermes billing through the executor (%s)", async scenario => { const previous = process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION; process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION = JSON.stringify([{ agent: "hermes", model: "fixture-model" }]); try { const onUsage = vi.fn(async (_receipt: import("@paperclipai/adapter-utils").AdapterUsageCheckpoint) => {}); const amount = scenario === "zero" ? 0 : 0.25; + const tokensMissing = scenario.endsWith("missing_tokens"); + const partialPrice = scenario.startsWith("partial"); const billing = { schema: "paperclip.usage.billing/v1", source: "provider_reported", biller: "openrouter", currency: "USD", - complete: scenario !== "partial", requestCount: 2, reportedRequestCount: scenario === "partial" ? 1 : 2, + complete: !partialPrice, requestCount: 2, reportedRequestCount: partialPrice ? 1 : 2, amountUsd: amount, amountUsdExact: amount.toFixed(9) }; const notification = (await import("@paperclipai/paperclip-runner/live")).rehydrateRunnerdUsageNotification({ - runDeltaAvailable: true, runDelta: { inputTokens: 20, outputTokens: 5, cacheReadTokens: 3, cacheWriteTokens: 2, providerCostUsd: amount }, + runDeltaAvailable: !tokensMissing, runDelta: { inputTokens: 20, outputTokens: 5, cacheReadTokens: 3, cacheWriteTokens: 2, providerCostUsd: amount }, ...(scenario === "legacy" ? {} : { billing }), }, "session", "turn"); const event = (eventType: string, sourceSeq: number, payload: object = {}) => ({ eventType, turnId: "turn", payload, @@ -5575,6 +5577,7 @@ describe("native terminal-turn accounting", () => { const usage = event("item.completed", 2, { kind: "usage", usage: notification.tokenUsage }); await accountingEvents.committed!(usage); expect(onUsage.mock.calls.at(-1)![0].complete).toBe(false); + expect(onUsage.mock.calls.at(-1)![0].settlement).toBeUndefined(); await accountingEvents.committed!(event("turn.completed", 3)); await accountingEvents.duplicate!(usage); return { result: { summary: "cancelled" }, terminal: { runTerminalState: "cancelled" }, turnId: "turn", @@ -5585,9 +5588,12 @@ describe("native terminal-turn accounting", () => { const result = await executePaperclipNativeSession({ db: leaseDb(), runnerInstanceId: "runner", onUsage, billingIdentity: { provider: "deepseek", biller: scenario === "wrong_account" ? "anthropic" : "openrouter", billingType: "metered_api" }, execution: { ...execution, provider: { kind: "acpx", agent: "hermes", model: "fixture-model", permissionMode: "approve-all", profile: QUALIFIED_ACPX_PROFILES.hermes } } as NativeExecutionInput }); - expect(result).toMatchObject({ usageComplete: true, costUsd: known ? amount : null, - costStatus: known && scenario !== "partial" ? "reported" : "unpriced" }); - expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: true, costUsd: result.costUsd, costStatus: result.costStatus })); + expect(result).toMatchObject({ usageComplete: !tokensMissing, costUsd: known ? amount : null, + costStatus: known && !partialPrice ? "reported" : "unpriced" }); + expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: !tokensMissing, costUsd: result.costUsd, costStatus: result.costStatus })); + if (known) expect(result.settlement).toEqual({ schema: "paperclip.accounting.settlement/v1", providerWorkEnded: true, usageComplete: !tokensMissing }); + else expect(result.settlement).toBeUndefined(); + if (tokensMissing) expect(result.usage).toBeUndefined(); if (known) expect(result.pricingProvenance).toMatchObject({ source: "provider_reported", version: "hermes-openrouter-wire/v1" }); } finally { if (previous === undefined) delete process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION; diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 2d4dc9af648..a366def81b5 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -7976,6 +7976,7 @@ async function executePaperclipNativeSessionWithinScope( let observedAccountingUsage: Record | null = null; let observedAccountingTurn: string | undefined; let nativeAccountingComplete = false; + let nativeSettlementReady = false; // OpenCode and ACPX report their current turn; Codex reports a cumulative run delta. // Rebuild per-turn state from the durable event log when a controller resumes. const selectedBiller = resolveNativeBilling(input.execution.provider, input.runnerEnvironment, input.billingIdentity).biller; @@ -7988,22 +7989,26 @@ async function executePaperclipNativeSessionWithinScope( )).orderBy(heartbeatRunEvents.seq); for (const row of history) turnAccounting.observe(record(row.payload).prpEvent as PrpEvent); } - const persistAccountingUsage = async (usage: Record | null, complete: boolean) => { + const persistAccountingUsage = async (usage: Record | null, complete: boolean, settlementReady = false) => { const accountingUsage = normalizeNativeUsage(usage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" }); - const accountingCost = accountingUsage ? nativeUsageCostUsd(usage, input.execution.provider, selectedBiller) : undefined; + const hermes = input.execution.provider.kind === "acpx" && input.execution.provider.agent === "hermes"; + const accountingCost = accountingUsage || hermes ? nativeUsageCostUsd(usage, input.execution.provider, selectedBiller) : undefined; complete = complete && accountingUsage !== undefined; - const costStatus = complete && usage?.accountingCostIncomplete !== true + const settlement = hermes && settlementReady ? { schema: "paperclip.accounting.settlement/v1" as const, providerWorkEnded: true as const, usageComplete: complete } : undefined; + const costStatus = (complete || settlementReady) && usage?.accountingCostIncomplete !== true ? input.execution.provider.kind === "aws_agentcore" ? "estimated" as const : nativeHermesPriceEvidence(usage, input.execution.provider, selectedBiller) ? "reported" as const : undefined : "unpriced" as const; const billing = resolveNativeBilling(input.execution.provider, input.runnerEnvironment, input.billingIdentity); const costUsdExact = typeof usage?.accountingCostUsdExact === "string" ? usage.accountingCostUsdExact : undefined; - await input.onUsage?.({ usage: accountingUsage, ...billing, complete, usageBasis: "per_run", costUsdExact, + await input.onUsage?.({ usage: accountingUsage, ...billing, complete, settlement, usageBasis: "per_run", costUsdExact, model: input.execution.provider.model ?? "unknown", costUsd: accountingCost ?? null, costStatus, ...nativeHermesPriceEvidence(usage, input.execution.provider, selectedBiller) }); if (!input.onUsage) await input.db.update(heartbeatRuns).set({ costAccountingPending: true, usageJson: sql`coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || ${JSON.stringify({ - ...accountingUsage, ...billing, accountingReceiptReady: complete, model: input.execution.provider.model ?? "unknown", + ...(settlement?.usageComplete === false ? { inputTokens: null, outputTokens: null, cachedInputTokens: null, cacheWriteTokens: null } : accountingUsage), + ...billing, accountingReceiptReady: complete || !!settlement, accountingSettlement: settlement ?? null, + accountingUsageComplete: complete, model: input.execution.provider.model ?? "unknown", costUsd: accountingCost ?? null, costUsdExact: costUsdExact ?? null, costStatus: costStatus ?? null, usageSource: "per_run", ...nativeHermesPriceEvidence(usage, input.execution.provider, selectedBiller), })}::jsonb`, @@ -8858,10 +8863,11 @@ async function executePaperclipNativeSessionWithinScope( const snapshot = turnAccounting.finish(native.turnId); observedAccountingUsage = snapshot.usage; nativeAccountingComplete = snapshot.complete && snapshot.turnId === native.turnId; + nativeSettlementReady = snapshot.settlementReady && snapshot.turnId === native.turnId; } else nativeAccountingComplete = observedAccountingUsage !== null && observedAccountingTurn !== undefined && observedAccountingTurn === native.turnId; native = { ...native, usage: observedAccountingUsage }; - await persistAccountingUsage(native.usage, nativeAccountingComplete); + await persistAccountingUsage(native.usage, nativeAccountingComplete, nativeSettlementReady); try { await completeManagedNativeCredentialTurn(managedCredentialSession); } catch { @@ -9504,13 +9510,14 @@ async function executePaperclipNativeSessionWithinScope( ...resolveNativeBilling(input.execution.provider, input.runnerEnvironment, input.billingIdentity), model: input.execution.provider.model, usage: normalizeNativeUsage(native.usage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" }), - costUsd: input.execution.provider.kind !== "openai_dot" && normalizeNativeUsage(native.usage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" }) + costUsd: input.execution.provider.kind !== "openai_dot" && ((input.execution.provider.kind === "acpx" && input.execution.provider.agent === "hermes") || normalizeNativeUsage(native.usage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" })) ? nativeUsageCostUsd(native.usage, input.execution.provider, selectedBiller) ?? null : null, costUsdExact: typeof native.usage?.accountingCostUsdExact === "string" ? native.usage.accountingCostUsdExact : undefined, - costStatus: nativeAccountingComplete && native.usage?.accountingCostIncomplete !== true + costStatus: (nativeAccountingComplete || nativeSettlementReady) && native.usage?.accountingCostIncomplete !== true ? input.execution.provider.kind === "aws_agentcore" ? "estimated" : nativeHermesPriceEvidence(native.usage, input.execution.provider, selectedBiller) ? "reported" : undefined : "unpriced", usageComplete: nativeAccountingComplete, + ...(nativeSettlementReady ? { settlement: { schema: "paperclip.accounting.settlement/v1" as const, providerWorkEnded: true as const, usageComplete: nativeAccountingComplete } } : {}), ...nativeHermesPriceEvidence(native.usage, input.execution.provider, selectedBiller), usageBasis: "per_run", nativeFinalization: finalization, @@ -9581,17 +9588,21 @@ export function nativeUsageBiller(provider: NativeExecutionInput["provider"]): s } export function createNativeTurnAccounting(provider: NativeExecutionInput["provider"], selectedBiller?: string | null) { + const hermes = provider.kind === "acpx" && provider.agent === "hermes"; const turns = new Map | null; terminal: boolean }>(); const sequences = new Map(); let currentTurn: string | undefined; const terminalTypes = ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"]; const snapshot = () => { - if (turns.size === 0) return { usage: null, turnId: undefined, complete: false }; + if (turns.size === 0) return { usage: null, turnId: undefined, complete: false, settlementReady: false }; let inputTokens = 0, outputTokens = 0, cacheReadTokens = 0, cacheWriteTokens = 0; let cost = 0n, costKnown = false, costIncomplete = false, complete = true; let requestCount = 0, reportedRequestCount = 0; + let tokensComplete = true; + let settlementReady = hermes; for (const [turnId, turn] of turns) { const normalized = normalizeNativeUsage(turn.usage); + tokensComplete &&= normalized !== undefined; complete &&= turnId !== "unattributed" && turn.terminal && normalized !== undefined; inputTokens += (normalized?.inputTokens ?? 0) - (normalized?.cacheWriteTokens ?? 0); outputTokens += normalized?.outputTokens ?? 0; @@ -9600,6 +9611,8 @@ export function createNativeTurnAccounting(provider: NativeExecutionInput["provi const price = nativeUsageCostUsd(turn.usage, provider, selectedBiller); const billing = provider.kind === "acpx" && provider.agent === "hermes" ? readProviderUsageBilling(turn.usage?.billing) : null; + settlementReady &&= turnId !== "unattributed" && turn.terminal && billing !== null + && selectedBiller === billing.biller && turn.usage !== null && nativeUsageMeasurement(turn.usage).providerCostUsd === billing.amountUsd; if (price === undefined) costIncomplete = true; else { cost += usdToUnits(billing?.amountUsdExact ?? price); costKnown = true; @@ -9618,14 +9631,15 @@ export function createNativeTurnAccounting(provider: NativeExecutionInput["provi ...(retainCost ? { providerCostUsd: Number(cost) / 1_000_000_000 } : {}) }; const current = currentTurn ? turns.get(currentTurn) : undefined; const costExact = `${cost / 1_000_000_000n}.${String(cost % 1_000_000_000n).padStart(9, "0")}`; - return { usage: { runDelta, accountingCostIncomplete: costIncomplete, + return { usage: { runDelta, runDeltaComplete: hermes ? tokensComplete : true, accountingCostIncomplete: costIncomplete, ...(retainCost && provider.kind === "acpx" && provider.agent === "hermes" ? { billing: { schema: "paperclip.usage.billing/v1", source: "provider_reported", biller: "openrouter", currency: "USD", complete: !costIncomplete, requestCount, reportedRequestCount, amountUsd: Number(costExact), amountUsdExact: costExact, } } : {}), ...(retainCost ? { accountingCostUsdExact: costExact } : {}) }, - turnId: current && normalizeNativeUsage(current.usage) ? currentTurn : undefined, complete }; + turnId: current && (normalizeNativeUsage(current.usage) || (settlementReady && readProviderUsageBilling(current.usage?.billing))) ? currentTurn : undefined, + complete, settlementReady }; }; return { observe(event: PrpEvent) { diff --git a/server/src/services/usage-receipts.test.ts b/server/src/services/usage-receipts.test.ts new file mode 100644 index 00000000000..f0535eaed7f --- /dev/null +++ b/server/src/services/usage-receipts.test.ts @@ -0,0 +1,104 @@ +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, companies, costEvents, createDb, heartbeatRuns, issues, nativeRunFinalizations, runUsageReceipts } from "@paperclipai/db"; +import type { AdapterUsageCheckpoint } from "@paperclipai/adapter-utils"; +import { startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { accountRunCost } from "./run-cost-accounting.js"; +import { createRunUsageRecorder, persistUsageReceipt, usageReceiptSpoolPath } from "./usage-receipts.js"; + +describe("closed native usage settlement", () => { + let temporary: Awaited>; + let db: ReturnType; + let home: string; + const previousHome = process.env.PAPERCLIP_HOME; + const companyId = randomUUID(), agentId = randomUUID(); + const settlement = { schema: "paperclip.accounting.settlement/v1", providerWorkEnded: true, usageComplete: false } as const; + const partial: AdapterUsageCheckpoint = { provider: "deepseek", biller: "openrouter", model: "fixture-model", + billingType: "metered_api", usageBasis: "per_run", complete: false, costStatus: "unpriced", + costUsd: 0.25, costUsdExact: "0.250000000", settlement }; + + beforeAll(async () => { + home = await mkdtemp(join(tmpdir(), "native-usage-settlement-")); + process.env.PAPERCLIP_HOME = home; + temporary = await startEmbeddedPostgresTestDatabase("native-usage-settlement-"); + db = createDb(temporary.connectionString); + await db.insert(companies).values({ id: companyId, name: "Usage settlement", issuePrefix: "USE" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Hermes", adapterType: "paperclip_runner", status: "idle" }); + }); + afterAll(async () => { + await temporary?.cleanup(); + if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; + else process.env.PAPERCLIP_HOME = previousHome; + if (home) await rm(home, { recursive: true, force: true }); + }); + async function createRun() { + const runId = randomUUID(), issueId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Settle attempted usage", status: "in_progress", assigneeAgentId: agentId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issueId, runtimeMode: "native", + status: "failed", finishedAt: new Date(), costAccountingPending: true, + usageJson: { ledgerScope: { issueId }, inputTokens: 99, outputTokens: 5 } }); + return { runId, issueId, recorder: await createRunUsageRecorder(db, { companyId, runId, adapterType: "paperclip_runner" }) }; + } + async function readRun(runId: string) { + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + return run; + } + + it("settles a retry's known subtotal exactly once after provider closure without inventing tokens", async () => { + const { runId, issueId, recorder } = await createRun(); + // A previous accepted request's counters are not the complete retry total. + await recorder.capture({ ...partial, settlement: undefined, usage: { inputTokens: 20, outputTokens: 2 } }); + const receipt = await recorder.complete({ exitCode: 1, signal: null, timedOut: false, ...partial, usageComplete: false }); + expect(receipt).toMatchObject({ complete: false, settlement, costUsdExact: "0.250000000" }); + expect(receipt.usage).toBeUndefined(); + const pending = await readRun(runId); + expect(pending.usageJson).toMatchObject({ inputTokens: null, outputTokens: null, accountingUsageComplete: false, accountingReceiptReady: true }); + await db.insert(nativeRunFinalizations).values({ runId, issueId, companyId, phase: "observed" }); + expect(await accountRunCost(db, runId)).toBe(false); + await db.update(nativeRunFinalizations).set({ phase: "terminal_failure" }).where(eq(nativeRunFinalizations.runId, runId)); + expect(await accountRunCost(db, runId)).toBe(true); + expect(await accountRunCost(db, runId)).toBe(false); + const [journal] = await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.id, String(pending.usageJson?.accountingReceiptId))); + await persistUsageReceipt(db, journal.receiptJson as Parameters[1]); + expect(await accountRunCost(db, runId)).toBe(false); + const events = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, runId)); + expect(events).toHaveLength(1); + expect(Number(events[0].costCents)).toBe(25); + expect(events[0].costStatus).toBe("unpriced"); + expect(await readRun(runId)).toMatchObject({ costAccountingPending: false, costAccountedAt: expect.any(Date) }); + }); + + it("keeps a legacy incomplete checkpoint pending without closure authority", async () => { + const { runId, recorder } = await createRun(); + await recorder.complete({ exitCode: 1, signal: null, timedOut: false, ...partial, settlement: undefined, usageComplete: false }); + expect(await accountRunCost(db, runId)).toBe(false); + expect(await readRun(runId)).toMatchObject({ costAccountingPending: true, costAccountedAt: null }); + }); + + it("does not use a closed later attempt to settle an unfinished attempt", async () => { + const { runId, recorder } = await createRun(); + await recorder.capture({ ...partial, attemptId: randomUUID(), settlement: undefined }); + const receipt = await recorder.capture({ ...partial, attemptId: randomUUID() }); + expect(receipt.settlement).toBeUndefined(); + expect(await accountRunCost(db, runId)).toBe(false); + }); + + it("cannot mint closure from an invalid settlement or a later capture failure", async () => { + const { runId, recorder } = await createRun(); + await expect(recorder.capture({ ...partial, settlement: { ...settlement, schema: "unknown/v1" } } as unknown as AdapterUsageCheckpoint)).rejects.toThrow(); + await recorder.complete({ exitCode: 1, signal: null, timedOut: false, ...partial, usageComplete: false }); + expect(await accountRunCost(db, runId)).toBe(false); + expect((await readRun(runId)).usageJson).toMatchObject({ accountingCaptureFailed: true, accountingReceiptReady: false }); + }); + + it("preserves v1 replay while rejecting settlement that disagrees with completeness", async () => { + const { runId, recorder } = await createRun(); + await expect(recorder.capture({ ...partial, complete: true })).rejects.toThrow("Settlement must preserve usage completeness"); + expect(await accountRunCost(db, runId)).toBe(false); + expect(usageReceiptSpoolPath().startsWith(home)).toBe(true); + }); +}); diff --git a/server/src/services/usage-receipts.ts b/server/src/services/usage-receipts.ts index 3606cd3a0ce..23dda0bab8f 100644 --- a/server/src/services/usage-receipts.ts +++ b/server/src/services/usage-receipts.ts @@ -30,7 +30,10 @@ const checkpointSchema = z.object({ pricingProvenance: pricingProvenanceSchema.optional(), costUsdExact: z.string().max(160).refine(value => { try { return !value.startsWith("-") && usdToUnits(value) >= 0n; } catch { return false; } }, "Invalid exact USD amount").nullable().optional(), providerRequestId: z.string().max(250).nullable().optional(), complete: z.boolean(), -}); + settlement: z.object({ schema: z.literal("paperclip.accounting.settlement/v1"), + providerWorkEnded: z.literal(true), usageComplete: z.boolean() }).strict().optional(), +}).refine(receipt => !receipt.settlement || receipt.settlement.usageComplete === receipt.complete, + "Settlement must preserve usage completeness"); const envelopeSchema = z.object({ schema: z.literal("paperclip/accounting-receipt/v1"), id: z.string().uuid(), companyId: z.string().uuid(), runId: z.string().uuid(), sourceId: z.string().uuid(), sequence: z.number().int().positive(), receivedAt: z.iso.datetime(), @@ -74,11 +77,13 @@ export async function persistUsageReceipt(db: Db, raw: UsageReceiptEnvelope) { await tx.update(heartbeatRuns).set({ costAccountingPending: true, accountingLastError: previous.accountingCaptureFailed === true ? "usage_capture_failed" : null, - usageJson: { ...previous, ...receipt.usage, + usageJson: { ...previous, ...(receipt.settlement?.usageComplete === false + ? { inputTokens: null, outputTokens: null, cachedInputTokens: null, cacheWriteTokens: null } : receipt.usage), provider: receipt.provider ?? "unknown", biller: receipt.biller ?? receipt.provider ?? "unknown", model: receipt.model ?? "unknown", billingType, costUsd: receipt.costUsd ?? null, costUsdExact: receipt.costUsdExact ?? null, cacheAdjustedCostUsd: receipt.cacheAdjustedCostUsd ?? null, usageByModel: receipt.usageByModel ?? null, usageSource: receipt.usageBasis ?? "per_run", providerRequestId: receipt.providerRequestId ?? null, - accountingReceiptReady: previous.accountingCaptureFailed !== true && receipt.complete && receipt.usageBasis !== "session_cumulative", accountingReceiptId: envelope.id, + accountingReceiptReady: previous.accountingCaptureFailed !== true && (receipt.complete || receipt.settlement?.providerWorkEnded === true) && receipt.usageBasis !== "session_cumulative", accountingReceiptId: envelope.id, + accountingSettlement: receipt.settlement ?? null, accountingUsageComplete: receipt.complete, accountingReceiptSequence: envelope.sequence, accountingReceiptReceivedAt: envelope.receivedAt, costStatus: receipt.costStatus ?? null, pricingProvenance: receipt.pricingProvenance ?? { source: receipt.costUsd != null || receipt.costUsdExact != null ? "provider_reported" : "unknown", version: "accounting-receipt/v1" }, @@ -238,7 +243,7 @@ export async function createRunUsageRecorder(db: Db, input: { companyId: string; const attempt = parsed.attemptId ?? currentAttempt; if (!attempts.has(attempt)) currentAttempt = attempt; const previous = attempts.get(attempt); - const priced = priceAnthropicReceipt(priceCodexReceipt({ ...parsed, usage: parsed.usage ?? previous?.usage, + const priced = priceAnthropicReceipt(priceCodexReceipt({ ...parsed, usage: parsed.settlement?.usageComplete === false ? undefined : parsed.usage ?? previous?.usage, complete: parsed.complete && !(previous?.usage && !parsed.usage) })); priced.pricingProvenance ??= { source: priced.costUsd != null || priced.costUsdExact != null || priced.cacheAdjustedCostUsd != null ? "provider_reported" : "unknown", @@ -281,6 +286,14 @@ export async function createRunUsageRecorder(db: Db, input: { companyId: string; // can be considered complete, even if a later attempt succeeded. receipt.complete = !captureFailed && parts.every(part => part.complete); } + // A closed provider boundary can settle a known subtotal while token or + // price totals remain unknown. Old incomplete checkpoints have no such + // authority; capture failure and an unfinished attempt still keep debt open. + if (parts.some(part => part.settlement)) { + receipt.settlement = !captureFailed && parts.every(part => part.complete || part.settlement?.providerWorkEnded === true) + ? { schema: "paperclip.accounting.settlement/v1", providerWorkEnded: true, usageComplete: receipt.complete } : undefined; + if (parts.some(part => part.settlement?.usageComplete === false)) receipt.usage = undefined; + } const envelope: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", id: randomUUID(), ...input, sourceId, sequence: ++sequence, receivedAt: new Date().toISOString(), receipt }; const file = await spoolUsageReceipt(envelope, directory); try { await persistUsageReceipt(db, envelope); await fs.rm(file, { force: true }); } @@ -298,7 +311,7 @@ export async function createRunUsageRecorder(db: Db, input: { companyId: string; // A bounded stdout tail can lose a terminal record already saved from the // full stream. Keep that stronger evidence, but never hide additional // usage/charges or certify a snapshot inherited from a replaced recorder. - if (sequence > 0 && previous?.complete && result.usageComplete === false) { + if (sequence > 0 && previous?.complete && result.usageComplete === false && !result.settlement) { const moreTokens = (["inputTokens", "cachedInputTokens", "cacheWriteTokens", "outputTokens"] as const) .some(key => (final.usage?.[key] ?? 0) > (previous.usage?.[key] ?? 0)); const finalPrice = final.cacheAdjustedCostUsd ?? final.costUsdExact ?? final.costUsd; diff --git a/tests/runner-e2e/hermes-api-connections.test.ts b/tests/runner-e2e/hermes-api-connections.test.ts index 62d8999f044..a372425ce54 100644 --- a/tests/runner-e2e/hermes-api-connections.test.ts +++ b/tests/runner-e2e/hermes-api-connections.test.ts @@ -43,7 +43,7 @@ describe("Hermes managed API connection qualification", () => { }); it.each(["valid", "reported-zero", "company-scope", "agent-scope", "run-scope", "task-scope", "paused-agent", "paused-company", "pause-reason", "budget-changed", "pending", "missing-settlement", "missing-price", "estimated", "partial", "wrong-biller", "wrong-provenance", "mismatched-exact", "unknown-tokens"])("calibrates public OpenRouter settlement: %s", async fault => { - const company: Record = { id: "company", status: "active", pauseReason: null, budgetMonthlyCents: 200 }; + const company: Record = { id: "company", status: "active", budgetMonthlyCents: 200 }; const agent: Record = { id: "agent", companyId: "company", status: "idle", pauseReason: null, budgetMonthlyCents: 200 }; const usage: Record = { biller: "openrouter", billingType: "metered_api", costStatus: "reported", costUsd: 0.0042, costUsdExact: "0.004200000", inputTokens: 40, outputTokens: 10, accountingReceiptReady: true, pricingProvenance: { source: "provider_reported", version: "hermes-openrouter-wire/v1" } }; diff --git a/tests/runner-e2e/hermes-api-connections.ts b/tests/runner-e2e/hermes-api-connections.ts index 56548c20e87..cd6a6f341ec 100644 --- a/tests/runner-e2e/hermes-api-connections.ts +++ b/tests/runner-e2e/hermes-api-connections.ts @@ -79,7 +79,7 @@ export async function captureHermesOpenRouterSettlement(input: { && typeof cost === "number" && Number.isFinite(cost) && cost >= 0 && typeof exact === "string" && /^(0|[1-9][0-9]{0,6})\.[0-9]{9}$/.test(exact) && Number(exact) === cost && typeof usage.inputTokens === "number" && Number.isSafeInteger(usage.inputTokens) && usage.inputTokens >= 0 && typeof usage.outputTokens === "number" && Number.isSafeInteger(usage.outputTokens) && usage.outputTokens >= 0 && usage.inputTokens + usage.outputTokens > 0 }, - { id: "budget-health-after-settlement", passed: company.status === "active" && company.pauseReason === null && agent.status === "idle" && agent.pauseReason === null + { id: "budget-health-after-settlement", passed: company.status === "active" && agent.status === "idle" && agent.pauseReason === null && company.budgetMonthlyCents === HERMES_API_CONNECTION_BUDGET_CENTS && agent.budgetMonthlyCents === HERMES_API_CONNECTION_BUDGET_CENTS }, ] }; }