diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index e87a8e96209..4c036b4c832 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -5457,6 +5457,56 @@ describe("native startup restart detachment", () => { }); describe("native terminal-turn accounting", () => { + it.each(["anthropic", "openai"].flatMap(biller => [false, true].flatMap(complete => + [false, true].map(shutdown => ({ biller, complete, shutdown })))))("settles direct Hermes $biller wire usage (complete: $complete, shutdown: $shutdown)", async ({ biller, complete, shutdown }) => { + const model = biller === "anthropic" ? "claude-haiku-4-5-20251001" : "gpt-6-luna"; + const previous = process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION; + process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION = JSON.stringify([{ agent: "hermes", model }]); + try { + const onUsage = vi.fn(async (_receipt: import("@paperclipai/adapter-utils").AdapterUsageCheckpoint) => {}); + const counts = { inputTokens: 12, outputTokens: 4, cacheReadTokens: 30, cacheWriteTokens: 20 }; + const tokenAccounting = { schema: "paperclip.usage.tokens/v1", source: "provider_wire", biller, model, + protocol: biller === "anthropic" ? "messages" : "chat_completions", complete, + requestCount: 2, reportedRequestCount: complete ? 2 : 1, tokens: counts, + pricingContext: { serviceTier: "standard", contextTier: "short" } }; + const notification = (await import("@paperclipai/paperclip-runner/live")).rehydrateRunnerdUsageNotification({ + runDeltaAvailable: complete, runDelta: { ...counts, providerCostUsd: 0 }, tokenAccounting, + }, "session", "turn"); + const usage: PrpEvent = { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceInstanceId: "provider", + sourceEventId: "provider:direct-usage:2", sourceSeq: 2, priority: 1, runId: execution.binding.runId, + normalizedSessionId: "session", turnId: "turn", eventType: "item.completed", emittedAt: new Date().toISOString(), + payload: { kind: "usage", usage: notification.tokenUsage } }; + state.execute.mockReset().mockImplementationOnce(async () => { + await accountingEvents.committed!({ ...usage, eventType: "turn.started", sourceSeq: 1, sourceEventId: "provider:direct-usage:1", payload: {} }); + if (!shutdown) { + await accountingEvents.committed!(usage); + await accountingEvents.committed!({ ...usage, eventType: "turn.completed", sourceSeq: 3, sourceEventId: "provider:direct-usage:3", payload: {} }); + } + return { result: { summary: "Finished native work" }, terminal: { runTerminalState: "succeeded" }, turnId: "turn", normalizedSessionId: "session", + providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: shutdown ? 1 : 3, highestContiguousSourceSeq: shutdown ? 1 : 3, + usage: null, ...(shutdown ? { settledUsageEvent: usage } : {}) }; + }); + const result = await executePaperclipNativeSession({ db: leaseDb(), runnerInstanceId: "runner", onUsage, + billingIdentity: { provider: biller, biller, billingType: "metered_api" }, execution: { ...execution, + provider: { kind: "acpx", agent: "hermes", model, permissionMode: "approve-all", profile: QUALIFIED_ACPX_PROFILES.hermes } } as NativeExecutionInput }); + expect(result).toMatchObject({ usageComplete: complete, costUsd: null, settlement: { providerWorkEnded: true, usageComplete: complete } }); + const checkpoint = onUsage.mock.calls.at(-1)![0]; + const priced = priceAnthropicReceipt(priceCodexReceipt(checkpoint)); + if (complete) { + expect(checkpoint).toMatchObject({ complete: true, provider: biller, biller, model, pricingContext: tokenAccounting.pricingContext, + usage: { inputTokens: 32, outputTokens: 4, cachedInputTokens: 30, cacheWriteTokens: 20 } }); + expect(priced.costStatus).toBe("estimated"); + expect(priced.pricingProvenance?.source).toBe("rate_card"); + } else { + expect(checkpoint).toMatchObject({ complete: false, costStatus: "unpriced" }); + expect(checkpoint.usage).toBeUndefined(); + expect(priced.costUsdExact).toBeUndefined(); + } + } finally { + if (previous === undefined) delete process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION; + else process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION = previous; + } + }); it.each([false, true])("prices only complete direct Claude API turn accounting (%s)", async complete => { const { rehydrateRunnerdUsageNotification } = await import("@paperclipai/paperclip-runner/live"); const { priceAnthropicReceipt } = await import("../anthropic-pricing.js"); @@ -5690,12 +5740,14 @@ describe("native terminal-turn accounting", () => { const result = await executePaperclipNativeSession({ db: leaseDb(), runnerInstanceId: "runner", onUsage, billingIdentity: { provider: "deepseek", biller: scenario === "wrong_account" ? "anthropic" : "openrouter", billingType: "metered_api" }, execution: { ...execution, provider: { kind: "acpx", agent: "hermes", model: "fixture-model", permissionMode: "approve-all", profile: QUALIFIED_ACPX_PROFILES.hermes } } as NativeExecutionInput }); - expect(result).toMatchObject({ usageComplete: !tokensMissing, costUsd: known ? amount : null, + // A foreign biller cannot certify the direct API's attempt tokens. + const tokensComplete = !tokensMissing && scenario !== "wrong_account"; + expect(result).toMatchObject({ usageComplete: tokensComplete, costUsd: known ? amount : null, costStatus: known && !partialPrice ? "reported" : "unpriced" }); - expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: !tokensMissing, costUsd: result.costUsd, costStatus: result.costStatus })); + expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: tokensComplete, costUsd: result.costUsd, costStatus: result.costStatus })); if (known) expect(result.settlement).toEqual({ schema: "paperclip.accounting.settlement/v1", providerWorkEnded: true, usageComplete: !tokensMissing }); else expect(result.settlement).toBeUndefined(); - if (tokensMissing) expect(result.usage).toBeUndefined(); + if (!tokensComplete) expect(result.usage).toBeUndefined(); if (known) expect(result.pricingProvenance).toMatchObject({ source: "provider_reported", version: "hermes-openrouter-wire/v1" }); } finally { if (previous === undefined) delete process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION; @@ -5734,9 +5786,12 @@ describe("native terminal-turn accounting", () => { const result = await executePaperclipNativeSession({ db: leaseDb(), runnerInstanceId: "runner", onUsage, billingIdentity: { provider: "deepseek", biller: scenario === "wrong_biller" ? "anthropic" : "openrouter", billingType: "metered_api" }, execution: { ...execution, provider: { kind: "acpx", agent: "hermes", model: "fixture-model", permissionMode: "approve-all", profile: QUALIFIED_ACPX_PROFILES.hermes } } as NativeExecutionInput }); - expect(result).toMatchObject({ usageComplete: true, costUsd: scenario === "wrong_biller" ? null : 0.25, - costStatus: scenario === "wrong_biller" ? "unpriced" : "reported", usage: { inputTokens: 31, outputTokens: 7 } }); - expect(onUsage.mock.calls.at(-1)![0]).toMatchObject({ costUsd: result.costUsd, complete: true }); + const tokensComplete = scenario !== "wrong_biller"; + expect(result).toMatchObject({ usageComplete: tokensComplete, costUsd: scenario === "wrong_biller" ? null : 0.25, + costStatus: scenario === "wrong_biller" ? "unpriced" : "reported" }); + if (tokensComplete) expect(result.usage).toMatchObject({ inputTokens: 31, outputTokens: 7 }); + else expect(result.usage).toBeUndefined(); + expect(onUsage.mock.calls.at(-1)![0]).toMatchObject({ costUsd: result.costUsd, complete: tokensComplete }); if (scenario === "wrong_biller") expect(result.settlement).toBeUndefined(); else expect(result.settlement).toMatchObject({ providerWorkEnded: true, usageComplete: true }); } finally {