test(hermes): verify API receipt authority through native execution

Require wrong-biller token totals to remain incomplete, and cover complete and interrupted Anthropic/OpenAI receipts during normal and shutdown delivery.

Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-09 10:29:27 -05:00
1 parent e025ff2286
commit f01d17e4fa
1 file changed
+61 -6
@@ -5457,6 +5457,56 @@ describe("native startup restart detachment", () => {
});
describe("native terminal-turn accounting", () => {
it.each(["anthropic", "openai"].flatMap(biller => [false, true].flatMap(complete =>
[false, true].map(shutdown => ({ biller, complete, shutdown })))))("settles direct Hermes $biller wire usage (complete: $complete, shutdown: $shutdown)", async ({ biller, complete, shutdown }) => {
const model = biller === "anthropic" ? "claude-haiku-4-5-20251001" : "gpt-6-luna";
const previous = process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION;
process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION = JSON.stringify([{ agent: "hermes", model }]);
try {
const onUsage = vi.fn(async (_receipt: import("@paperclipai/adapter-utils").AdapterUsageCheckpoint) => {});
const counts = { inputTokens: 12, outputTokens: 4, cacheReadTokens: 30, cacheWriteTokens: 20 };
const tokenAccounting = { schema: "paperclip.usage.tokens/v1", source: "provider_wire", biller, model,
protocol: biller === "anthropic" ? "messages" : "chat_completions", complete,
requestCount: 2, reportedRequestCount: complete ? 2 : 1, tokens: counts,
pricingContext: { serviceTier: "standard", contextTier: "short" } };
const notification = (await import("@paperclipai/paperclip-runner/live")).rehydrateRunnerdUsageNotification({
runDeltaAvailable: complete, runDelta: { ...counts, providerCostUsd: 0 }, tokenAccounting,
}, "session", "turn");
const usage: PrpEvent = { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceInstanceId: "provider",
sourceEventId: "provider:direct-usage:2", sourceSeq: 2, priority: 1, runId: execution.binding.runId,
normalizedSessionId: "session", turnId: "turn", eventType: "item.completed", emittedAt: new Date().toISOString(),
payload: { kind: "usage", usage: notification.tokenUsage } };
state.execute.mockReset().mockImplementationOnce(async () => {
await accountingEvents.committed!({ ...usage, eventType: "turn.started", sourceSeq: 1, sourceEventId: "provider:direct-usage:1", payload: {} });
if (!shutdown) {
await accountingEvents.committed!(usage);
await accountingEvents.committed!({ ...usage, eventType: "turn.completed", sourceSeq: 3, sourceEventId: "provider:direct-usage:3", payload: {} });
}
return { result: { summary: "Finished native work" }, terminal: { runTerminalState: "succeeded" }, turnId: "turn", normalizedSessionId: "session",
providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: shutdown ? 1 : 3, highestContiguousSourceSeq: shutdown ? 1 : 3,
usage: null, ...(shutdown ? { settledUsageEvent: usage } : {}) };
});
const result = await executePaperclipNativeSession({ db: leaseDb(), runnerInstanceId: "runner", onUsage,
billingIdentity: { provider: biller, biller, billingType: "metered_api" }, execution: { ...execution,
provider: { kind: "acpx", agent: "hermes", model, permissionMode: "approve-all", profile: QUALIFIED_ACPX_PROFILES.hermes } } as NativeExecutionInput });
expect(result).toMatchObject({ usageComplete: complete, costUsd: null, settlement: { providerWorkEnded: true, usageComplete: complete } });
const checkpoint = onUsage.mock.calls.at(-1)![0];
const priced = priceAnthropicReceipt(priceCodexReceipt(checkpoint));
if (complete) {
expect(checkpoint).toMatchObject({ complete: true, provider: biller, biller, model, pricingContext: tokenAccounting.pricingContext,
usage: { inputTokens: 32, outputTokens: 4, cachedInputTokens: 30, cacheWriteTokens: 20 } });
expect(priced.costStatus).toBe("estimated");
expect(priced.pricingProvenance?.source).toBe("rate_card");
} else {
expect(checkpoint).toMatchObject({ complete: false, costStatus: "unpriced" });
expect(checkpoint.usage).toBeUndefined();
expect(priced.costUsdExact).toBeUndefined();
}
} finally {
if (previous === undefined) delete process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION;
else process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION = previous;
}
});
it.each([false, true])("prices only complete direct Claude API turn accounting (%s)", async complete => {
const { rehydrateRunnerdUsageNotification } = await import("@paperclipai/paperclip-runner/live");
const { priceAnthropicReceipt } = await import("../anthropic-pricing.js");
@@ -5690,12 +5740,14 @@ describe("native terminal-turn accounting", () => {
const result = await executePaperclipNativeSession({ db: leaseDb(), runnerInstanceId: "runner", onUsage,
billingIdentity: { provider: "deepseek", biller: scenario === "wrong_account" ? "anthropic" : "openrouter", billingType: "metered_api" },
execution: { ...execution, provider: { kind: "acpx", agent: "hermes", model: "fixture-model", permissionMode: "approve-all", profile: QUALIFIED_ACPX_PROFILES.hermes } } as NativeExecutionInput });
expect(result).toMatchObject({ usageComplete: !tokensMissing, costUsd: known ? amount : null,
// A foreign biller cannot certify the direct API's attempt tokens.
const tokensComplete = !tokensMissing && scenario !== "wrong_account";
expect(result).toMatchObject({ usageComplete: tokensComplete, costUsd: known ? amount : null,
costStatus: known && !partialPrice ? "reported" : "unpriced" });
expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: !tokensMissing, costUsd: result.costUsd, costStatus: result.costStatus }));
expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: tokensComplete, costUsd: result.costUsd, costStatus: result.costStatus }));
if (known) expect(result.settlement).toEqual({ schema: "paperclip.accounting.settlement/v1", providerWorkEnded: true, usageComplete: !tokensMissing });
else expect(result.settlement).toBeUndefined();
if (tokensMissing) expect(result.usage).toBeUndefined();
if (!tokensComplete) expect(result.usage).toBeUndefined();
if (known) expect(result.pricingProvenance).toMatchObject({ source: "provider_reported", version: "hermes-openrouter-wire/v1" });
} finally {
if (previous === undefined) delete process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION;
@@ -5734,9 +5786,12 @@ describe("native terminal-turn accounting", () => {
const result = await executePaperclipNativeSession({ db: leaseDb(), runnerInstanceId: "runner", onUsage,
billingIdentity: { provider: "deepseek", biller: scenario === "wrong_biller" ? "anthropic" : "openrouter", billingType: "metered_api" },
execution: { ...execution, provider: { kind: "acpx", agent: "hermes", model: "fixture-model", permissionMode: "approve-all", profile: QUALIFIED_ACPX_PROFILES.hermes } } as NativeExecutionInput });
expect(result).toMatchObject({ usageComplete: true, costUsd: scenario === "wrong_biller" ? null : 0.25,
costStatus: scenario === "wrong_biller" ? "unpriced" : "reported", usage: { inputTokens: 31, outputTokens: 7 } });
expect(onUsage.mock.calls.at(-1)![0]).toMatchObject({ costUsd: result.costUsd, complete: true });
const tokensComplete = scenario !== "wrong_biller";
expect(result).toMatchObject({ usageComplete: tokensComplete, costUsd: scenario === "wrong_biller" ? null : 0.25,
costStatus: scenario === "wrong_biller" ? "unpriced" : "reported" });
if (tokensComplete) expect(result.usage).toMatchObject({ inputTokens: 31, outputTokens: 7 });
else expect(result.usage).toBeUndefined();
expect(onUsage.mock.calls.at(-1)![0]).toMatchObject({ costUsd: result.costUsd, complete: tokensComplete });
if (scenario === "wrong_biller") expect(result.settlement).toBeUndefined();
else expect(result.settlement).toMatchObject({ providerWorkEnded: true, usageComplete: true });
} finally {