mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-18 15:39:47 +02:00
test(hermes): verify API receipt authority through native execution
Require wrong-biller token totals to remain incomplete, and cover complete and interrupted Anthropic/OpenAI receipts during normal and shutdown delivery. Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
e025ff2286
commit
f01d17e4fa
1 file changed
+61
-6
@@ -5457,6 +5457,56 @@ describe("native startup restart detachment", () => {
|
||||
});
|
||||
|
||||
describe("native terminal-turn accounting", () => {
|
||||
it.each(["anthropic", "openai"].flatMap(biller => [false, true].flatMap(complete =>
|
||||
[false, true].map(shutdown => ({ biller, complete, shutdown })))))("settles direct Hermes $biller wire usage (complete: $complete, shutdown: $shutdown)", async ({ biller, complete, shutdown }) => {
|
||||
const model = biller === "anthropic" ? "claude-haiku-4-5-20251001" : "gpt-6-luna";
|
||||
const previous = process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION;
|
||||
process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION = JSON.stringify([{ agent: "hermes", model }]);
|
||||
try {
|
||||
const onUsage = vi.fn(async (_receipt: import("@paperclipai/adapter-utils").AdapterUsageCheckpoint) => {});
|
||||
const counts = { inputTokens: 12, outputTokens: 4, cacheReadTokens: 30, cacheWriteTokens: 20 };
|
||||
const tokenAccounting = { schema: "paperclip.usage.tokens/v1", source: "provider_wire", biller, model,
|
||||
protocol: biller === "anthropic" ? "messages" : "chat_completions", complete,
|
||||
requestCount: 2, reportedRequestCount: complete ? 2 : 1, tokens: counts,
|
||||
pricingContext: { serviceTier: "standard", contextTier: "short" } };
|
||||
const notification = (await import("@paperclipai/paperclip-runner/live")).rehydrateRunnerdUsageNotification({
|
||||
runDeltaAvailable: complete, runDelta: { ...counts, providerCostUsd: 0 }, tokenAccounting,
|
||||
}, "session", "turn");
|
||||
const usage: PrpEvent = { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceInstanceId: "provider",
|
||||
sourceEventId: "provider:direct-usage:2", sourceSeq: 2, priority: 1, runId: execution.binding.runId,
|
||||
normalizedSessionId: "session", turnId: "turn", eventType: "item.completed", emittedAt: new Date().toISOString(),
|
||||
payload: { kind: "usage", usage: notification.tokenUsage } };
|
||||
state.execute.mockReset().mockImplementationOnce(async () => {
|
||||
await accountingEvents.committed!({ ...usage, eventType: "turn.started", sourceSeq: 1, sourceEventId: "provider:direct-usage:1", payload: {} });
|
||||
if (!shutdown) {
|
||||
await accountingEvents.committed!(usage);
|
||||
await accountingEvents.committed!({ ...usage, eventType: "turn.completed", sourceSeq: 3, sourceEventId: "provider:direct-usage:3", payload: {} });
|
||||
}
|
||||
return { result: { summary: "Finished native work" }, terminal: { runTerminalState: "succeeded" }, turnId: "turn", normalizedSessionId: "session",
|
||||
providerSessionId: null, driverKind: "test", driverVersion: "1", nativeEventCount: shutdown ? 1 : 3, highestContiguousSourceSeq: shutdown ? 1 : 3,
|
||||
usage: null, ...(shutdown ? { settledUsageEvent: usage } : {}) };
|
||||
});
|
||||
const result = await executePaperclipNativeSession({ db: leaseDb(), runnerInstanceId: "runner", onUsage,
|
||||
billingIdentity: { provider: biller, biller, billingType: "metered_api" }, execution: { ...execution,
|
||||
provider: { kind: "acpx", agent: "hermes", model, permissionMode: "approve-all", profile: QUALIFIED_ACPX_PROFILES.hermes } } as NativeExecutionInput });
|
||||
expect(result).toMatchObject({ usageComplete: complete, costUsd: null, settlement: { providerWorkEnded: true, usageComplete: complete } });
|
||||
const checkpoint = onUsage.mock.calls.at(-1)![0];
|
||||
const priced = priceAnthropicReceipt(priceCodexReceipt(checkpoint));
|
||||
if (complete) {
|
||||
expect(checkpoint).toMatchObject({ complete: true, provider: biller, biller, model, pricingContext: tokenAccounting.pricingContext,
|
||||
usage: { inputTokens: 32, outputTokens: 4, cachedInputTokens: 30, cacheWriteTokens: 20 } });
|
||||
expect(priced.costStatus).toBe("estimated");
|
||||
expect(priced.pricingProvenance?.source).toBe("rate_card");
|
||||
} else {
|
||||
expect(checkpoint).toMatchObject({ complete: false, costStatus: "unpriced" });
|
||||
expect(checkpoint.usage).toBeUndefined();
|
||||
expect(priced.costUsdExact).toBeUndefined();
|
||||
}
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION;
|
||||
else process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION = previous;
|
||||
}
|
||||
});
|
||||
it.each([false, true])("prices only complete direct Claude API turn accounting (%s)", async complete => {
|
||||
const { rehydrateRunnerdUsageNotification } = await import("@paperclipai/paperclip-runner/live");
|
||||
const { priceAnthropicReceipt } = await import("../anthropic-pricing.js");
|
||||
@@ -5690,12 +5740,14 @@ describe("native terminal-turn accounting", () => {
|
||||
const result = await executePaperclipNativeSession({ db: leaseDb(), runnerInstanceId: "runner", onUsage,
|
||||
billingIdentity: { provider: "deepseek", biller: scenario === "wrong_account" ? "anthropic" : "openrouter", billingType: "metered_api" },
|
||||
execution: { ...execution, provider: { kind: "acpx", agent: "hermes", model: "fixture-model", permissionMode: "approve-all", profile: QUALIFIED_ACPX_PROFILES.hermes } } as NativeExecutionInput });
|
||||
expect(result).toMatchObject({ usageComplete: !tokensMissing, costUsd: known ? amount : null,
|
||||
// A foreign biller cannot certify the direct API's attempt tokens.
|
||||
const tokensComplete = !tokensMissing && scenario !== "wrong_account";
|
||||
expect(result).toMatchObject({ usageComplete: tokensComplete, costUsd: known ? amount : null,
|
||||
costStatus: known && !partialPrice ? "reported" : "unpriced" });
|
||||
expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: !tokensMissing, costUsd: result.costUsd, costStatus: result.costStatus }));
|
||||
expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: tokensComplete, costUsd: result.costUsd, costStatus: result.costStatus }));
|
||||
if (known) expect(result.settlement).toEqual({ schema: "paperclip.accounting.settlement/v1", providerWorkEnded: true, usageComplete: !tokensMissing });
|
||||
else expect(result.settlement).toBeUndefined();
|
||||
if (tokensMissing) expect(result.usage).toBeUndefined();
|
||||
if (!tokensComplete) expect(result.usage).toBeUndefined();
|
||||
if (known) expect(result.pricingProvenance).toMatchObject({ source: "provider_reported", version: "hermes-openrouter-wire/v1" });
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION;
|
||||
@@ -5734,9 +5786,12 @@ describe("native terminal-turn accounting", () => {
|
||||
const result = await executePaperclipNativeSession({ db: leaseDb(), runnerInstanceId: "runner", onUsage,
|
||||
billingIdentity: { provider: "deepseek", biller: scenario === "wrong_biller" ? "anthropic" : "openrouter", billingType: "metered_api" },
|
||||
execution: { ...execution, provider: { kind: "acpx", agent: "hermes", model: "fixture-model", permissionMode: "approve-all", profile: QUALIFIED_ACPX_PROFILES.hermes } } as NativeExecutionInput });
|
||||
expect(result).toMatchObject({ usageComplete: true, costUsd: scenario === "wrong_biller" ? null : 0.25,
|
||||
costStatus: scenario === "wrong_biller" ? "unpriced" : "reported", usage: { inputTokens: 31, outputTokens: 7 } });
|
||||
expect(onUsage.mock.calls.at(-1)![0]).toMatchObject({ costUsd: result.costUsd, complete: true });
|
||||
const tokensComplete = scenario !== "wrong_biller";
|
||||
expect(result).toMatchObject({ usageComplete: tokensComplete, costUsd: scenario === "wrong_biller" ? null : 0.25,
|
||||
costStatus: scenario === "wrong_biller" ? "unpriced" : "reported" });
|
||||
if (tokensComplete) expect(result.usage).toMatchObject({ inputTokens: 31, outputTokens: 7 });
|
||||
else expect(result.usage).toBeUndefined();
|
||||
expect(onUsage.mock.calls.at(-1)![0]).toMatchObject({ costUsd: result.costUsd, complete: tokensComplete });
|
||||
if (scenario === "wrong_biller") expect(result.settlement).toBeUndefined();
|
||||
else expect(result.settlement).toMatchObject({ providerWorkEnded: true, usageComplete: true });
|
||||
} finally {
|
||||
|
||||
Reference in new issue
Block a user