diff --git a/doc/plans/2026-10-06-hermes-native-runner.md b/doc/plans/2026-10-06-hermes-native-runner.md index 24f905aa084..02e50f5d365 100644 --- a/doc/plans/2026-10-06-hermes-native-runner.md +++ b/doc/plans/2026-10-06-hermes-native-runner.md @@ -954,3 +954,46 @@ The adapter failure-path suite passes all 31 tests after the build completes; the earlier concurrent-build test failures remain in their original log. This is offline and deterministic proof. A new clean-head live campaign and fresh CI/review remain required. Docker is confirmed stopped. + +### 2026-10-08 input-yield accounting correction + +At `d968e1d002`, all three stacked PRs passed their checks and fresh review. +The Linux native transport job passed against the current closure in 99.743 +seconds using a deterministic loopback model. That is Linux transport proof, +not execution in Daytona or paid-provider qualification. + +The Mac OpenRouter completion campaign passed 15 checks in 44.170 seconds, +with cleanup and reported settled cost of `$0.000645327`. The question retry +passed six behavioral checks in 91.154 seconds, but its first, input-yield run +had pending unpriced accounting. Its continuation emitted a reported complete +`$0.002543140` wire receipt. The original results remain unchanged; the question +behavioral pass does not qualify billing. An earlier question attempt failed +during provider startup and remains a separate infrastructure failure. + +The transcript consumer revokes tool authority at a governed wait before +Hermes emits terminal usage. The per-turn runtime now reads the last bound +usage fact after owned shutdown and notification drainage, then journals it +through the existing accounting path. Session baselines cannot substitute. +Wrong session/run/turn/source, missing receipts and timed-out reads retain +unknown accounting. The versioned extended-harnesses oracle now requires every +run in a Hermes/OpenRouter workflow to settle reported cost with healthy +200-cent budgets before cleanup. + +Focused proof passes 258 Runner tests, including shutdown retry and bounded +read failure, and 613 native-executor tests, including duplicate receipts and +wrong biller, plus five durable-journal tests and 64 Codex lifecycle/recovery +regression tests. Product E2E oracle tests pass 88 checks. The production +Rust/image fixture passes in 17.850 seconds using the retained binary. The +Python and ACPX/control/restore fixture passes in 110.567 seconds. Runner, +server and Product E2E TypeScript and generated contracts pass. These use a +deterministic loopback model. Clean-head live question accounting, full cloud +checks and review remain required for this correction. A full API-authority +guard implicitly rebuilt Rust before hitting a loopback sandbox error. Its +failed log is retained, and 828,116,992 bytes of unused cache were removed. +All subsequent full build guards run in cloud CI. Docker remains stopped. +The 29 authenticated Runner API integration tests pass separately with the +retained binary and loopback access. Package-boundary checks pass. Optional +forbidden/tracked-import guards still report existing violations in unchanged +files; their failed logs are retained. These are not passing guard results. +The total reserved amount remains `$23` against the `$25` cap; unknown spend +retains its reservation. Hermes remains pending qualification. diff --git a/packages/paperclip-runner/docs/hermes.md b/packages/paperclip-runner/docs/hermes.md index b5fb5a3b037..4a222f12855 100644 --- a/packages/paperclip-runner/docs/hermes.md +++ b/packages/paperclip-runner/docs/hermes.md @@ -125,6 +125,14 @@ extend the task's execution deadline. are incomplete. Missing token totals stay unknown, incomplete charges stay unpriced, and the existing controller finalization and capture-failure fences still control acknowledgement. Older incomplete receipts remain pending. +- A governed input wait can stop transcript consumption before native usage + arrives. For Hermes's per-turn lifecycle, the runtime joins the authenticated + notification mapper after owned shutdown and reads its last usage event. + This read-only fact must match the original runner, session, run and turn, + then enters the existing durable accounting journal. It cannot reopen tool + authority. Missing, invalid or timed-out reads do not invent usage or cost. + This optional execution-result field carries an existing v1 PRP event; it + does not change persisted execution inputs or the Rust wire contract. - Paperclip owns task titles. Hermes keeps its immediate derived session title; its paid background title upgrade is disabled in the managed profile so it cannot start inference after a turn's usage receipt has closed. diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.ts index bf5496c097e..a33ae9abbc1 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.ts @@ -866,6 +866,10 @@ class HarnessNativeSession implements NativeSession { return this.#withProtocolIntegrity(() => this.#session.usage?.() ?? null); } + async accountingUsageEvent(): Promise { + return this.#withProtocolIntegrity(() => this.#session.accountingUsageEvent?.() ?? null); + } + close(input: { reason: string }) { return this.#session.close(input); } diff --git a/packages/paperclip-runner/src/contracts/harness-driver.ts b/packages/paperclip-runner/src/contracts/harness-driver.ts index ee04ce4352b..443fdc6e47a 100644 --- a/packages/paperclip-runner/src/contracts/harness-driver.ts +++ b/packages/paperclip-runner/src/contracts/harness-driver.ts @@ -544,6 +544,8 @@ export interface HarnessSession { read?(): Promise>; reconcile?(): Promise>; usage?(): Promise | null>; + /** Read the last bound usage fact, including a passive shutdown suffix. */ + accountingUsageEvent?(): Promise; transcript?(): Promise; snapshot(): Promise; /** Relinquish controller authority without semantically closing the session. */ diff --git a/packages/paperclip-runner/src/contracts/native-execution.ts b/packages/paperclip-runner/src/contracts/native-execution.ts index 54404b37a65..9000a19e5e0 100644 --- a/packages/paperclip-runner/src/contracts/native-execution.ts +++ b/packages/paperclip-runner/src/contracts/native-execution.ts @@ -3,7 +3,7 @@ import { isSupportedAcpxProfileVersion, type AcpxProfileVersion } from "../drive import { isProviderMode } from "./provider-mode.js"; import { createHash } from "node:crypto"; import { parseNativeUserAttachments, type NativeUserAttachment } from "./user-attachments.js"; -import type { PrpStructuredRunResult, PrpTerminalState } from "../protocol/replay-contract.js"; +import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "../protocol/replay-contract.js"; import { explicitTaskSkillNames, parseNativeRuntimeContext, type NativeRuntimeContextSnapshot } from "./runtime-context.js"; export const NATIVE_EXECUTION_INPUT_SCHEMA_V1 = "paperclip.native-execution-input.v1" as const; @@ -287,6 +287,8 @@ export interface NativeSessionExecutionResult { nativeEventCount: number; highestContiguousSourceSeq: number; usage: Record | null; + /** Optional v1 PRP usage fact retained after verified per-turn shutdown. */ + settledUsageEvent?: PrpEvent; /** The active durable goal reached a safe turn boundary for run rollover. */ goalRolloverRequired?: boolean; } diff --git a/packages/paperclip-runner/src/contracts/native-session-backend.ts b/packages/paperclip-runner/src/contracts/native-session-backend.ts index 87af9e2ad05..b43480d3fb8 100644 --- a/packages/paperclip-runner/src/contracts/native-session-backend.ts +++ b/packages/paperclip-runner/src/contracts/native-session-backend.ts @@ -228,6 +228,8 @@ export interface NativeSession { turnId: string | null; } | null>; usage?(): Promise | null>; + /** Read-only accounting fact. Call only after the exact close owner succeeds. */ + accountingUsageEvent?(): Promise; snapshot( options?: NativeSessionSnapshotOptions, ): Promise; diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts index c4953f5cc1b..de18908c84b 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts @@ -765,6 +765,7 @@ class CodexAcpxSession implements HarnessSession { turnId: string; } | null = null; #usage: Record | null = null; + #accountingUsageEvent: PrpEvent | null = null; #assistantText = ""; #assistantMessageId: string | null = null; #closed = false; @@ -885,6 +886,7 @@ class CodexAcpxSession implements HarnessSession { } const turnId = `turn-${randomBytes(12).toString("hex")}`; this.#activeTurnId = turnId; + this.#accountingUsageEvent = null; this.#turnControls.begin(turnId); this.#assistantText = ""; this.#assistantMessageId = null; @@ -1251,6 +1253,10 @@ class CodexAcpxSession implements HarnessSession { return this.#usage === null ? null : structuredClone(this.#usage); } + async accountingUsageEvent(): Promise { + return this.#accountingUsageEvent === null ? null : structuredClone(this.#accountingUsageEvent); + } + async transcript(): Promise { return { schema: "paperclip-runner/harness-transcript/v1", @@ -2000,6 +2006,8 @@ class CodexAcpxSession implements HarnessSession { return false; } this.#sourceSequence = sourceSeq; + if (eventType === "item.completed" && payload.kind === "usage") + this.#accountingUsageEvent = structuredClone(event); this.#retainTranscriptEvent(event); return true; } diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.events.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.events.test.ts index fbf45dfea9f..610a5690f38 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.events.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.events.test.ts @@ -350,6 +350,32 @@ describe("Codex app-server Codex driver", () => { }); }); + it("retains bound usage from shutdown after the transcript consumer relinquishes ownership", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-shutdown-usage", normalizedSessionId: "normalized-shutdown-usage", workingDirectory: WORKSPACE, + }); + const turn = await session.startTurn({ message: { role: "user", text: "Ask a question." } }); + transport.push("turn/started", { threadId: "thread-1", turn: { id: turn.turnId, status: "inProgress" } }); + const originalClose = transport.close.bind(transport); + transport.close = async () => { + transport.push("thread/tokenUsage/updated", { threadId: "thread-1", turnId: turn.turnId, + tokenUsage: { total: { inputTokens: 31, outputTokens: 7 } } }); + transport.push("turn/completed", { threadId: "thread-1", turn: { id: turn.turnId, status: "interrupted", items: [] } }); + await originalClose(); + }; + // The public event queue closes before transport.close emits its suffix. + expect(await session.accountingUsageEvent?.()).toBeNull(); + await session.close({ reason: "durable question yield" }); + const receipt = await session.accountingUsageEvent?.(); + expect(receipt).toMatchObject({ sourceKind: "runner", runId: "run-shutdown-usage", + normalizedSessionId: "normalized-shutdown-usage", turnId: turn.turnId, + eventType: "item.completed", payload: { kind: "usage", usage: { runDelta: { inputTokens: 31, outputTokens: 7 }, runDeltaComplete: true } } }); + expect(validatePrpEvent(receipt).ok).toBe(true); + receipt!.payload.kind = "mutated"; + expect((await session.accountingUsageEvent?.())?.payload.kind).toBe("usage"); + }); + it("normalizes provider message and reasoning phases onto every streamed item event", async () => { const transport = new FakeCodexTransport(); const session = await makeDriver([transport]).openSession({ diff --git a/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts b/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts index 48227d2af68..d42128c118b 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts @@ -59,6 +59,8 @@ export class CodexHarnessSession extends CodexSessionState implements HarnessSession { + readonly #notificationPump: Promise; + constructor(input: CodexSessionStateInput) { super(input); this.transport.setServerRequestHandler((request) => @@ -67,8 +69,9 @@ export class CodexHarnessSession initializeCodexSessionEvents(this, input); if (this.terminal) { this.eventQueue.close(); + this.#notificationPump = Promise.resolve(); } else { - void pumpNotifications(this); + this.#notificationPump = pumpNotifications(this); } } @@ -119,6 +122,7 @@ export class CodexHarnessSession this.usageSnapshot = codexRunUsage(this.codexUsageBaseline); } this.runId = input.runId; + this.lastAccountingUsageEvent = null; this.result = null; this.resultFingerprint = null; this.resultCallId = null; @@ -783,6 +787,12 @@ export class CodexHarnessSession : structuredClone(this.usageSnapshot); } + async accountingUsageEvent(): Promise { + this.assertProtocolIntegrity(); + return this.lastAccountingUsageEvent === null + ? null : structuredClone(this.lastAccountingUsageEvent); + } + async snapshot(): Promise { this.assertProtocolIntegrity(); return { @@ -826,6 +836,9 @@ export class CodexHarnessSession this.cancelPendingRequests("session_closed"); this.eventQueue.close(); await this.transport.close(input?.reason); + // Shutdown drains the authenticated provider tail. Join its mapper so a + // late usage frame cannot be observed only after close has returned. + await this.#notificationPump; } async detachControllerForRestart(): Promise { diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts index 318ad4cc7b5..d4d211d42c1 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts @@ -109,6 +109,7 @@ export class CodexSessionState { sourceSequence: number; activeTurnId: string | null; usageSnapshot: Record | null = null; + lastAccountingUsageEvent: PrpEvent | null = null; codexUsageBaseline: CodexUsageBaseline | null = null; result: PrpStructuredRunResult | null = null; resultFingerprint: string | null = null; @@ -435,7 +436,7 @@ export class CodexSessionState { refs: { turnId?: string; itemId?: string } = {}, ): void { const sourceSeq = ++this.sourceSequence; - this.eventQueue.push({ + const event: PrpEvent = { schema: "paperclip.prp.event.v1", sourceEventId: `${this.runnerInstanceId}:${this.runId}:${sourceSeq}`, sourceSeq, @@ -450,7 +451,12 @@ export class CodexSessionState { priority: eventType === "run.result.proposed" ? 0 : 1, emittedAt: this.now().toISOString(), payload, - }); + }; + // A governed wait revokes accepted output before transport shutdown. + // Keep only its usage fact independently of the closed transcript queue. + if (eventType === "item.completed" && payload.kind === "usage") + this.lastAccountingUsageEvent = structuredClone(event); + this.eventQueue.push(event); } emitGoalCapabilities(): void { diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index db0344becc7..f36ee65c4db 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { QUALIFIED_ACPX_PROFILES } from "./drivers/acpx/qualified-profiles.js"; import { describe, expect, it, vi } from "vitest"; import type { ControlPlanePort } from "./contracts/control-plane-port.js"; @@ -225,6 +226,64 @@ function highestContiguous(events: PrpEvent[]): number { } describe("executeNativeSession recovery", () => { + it.each(["bound", "close_retry", "unavailable", "wrong_run", "wrong_session", "wrong_turn", "wrong_source", "tool_event", "invalid", "warm", "legacy_provider"])( + "reads a passive shutdown receipt only for its closed Hermes turn (%s)", async scenario => { + let closed = false; + const usageEvent: PrpEvent = runnerEvent(3, "item.completed", { kind: "usage", usage: { + runDelta: { inputTokens: 31, outputTokens: 7, providerCostUsd: 0.25 }, runDeltaComplete: true, + } }); + if (scenario === "wrong_run") usageEvent.runId = "another-run"; + if (scenario === "wrong_session") usageEvent.normalizedSessionId = "another-session"; + if (scenario === "wrong_turn") usageEvent.turnId = "another-turn"; + if (scenario === "wrong_source") usageEvent.sourceInstanceId = "another-source"; + if (scenario === "tool_event") usageEvent.payload.kind = "dynamicToolCall"; + if (scenario === "invalid") usageEvent.schemaVersion = 99; + const read = vi.fn(async () => { expect(closed).toBe(true); + if (scenario === "unavailable") return await new Promise(() => undefined); + return usageEvent; }); + let closeAttempts = 0; + const session: NativeSession = { + identity: () => identity, + async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, + async *events() { yield runnerEvent(1, "turn.started"); yield runnerEvent(2, "turn.completed"); }, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { return { result: yieldedResult, terminal: { ...terminal, reportedWorkDisposition: "yielded" }, turnId: "turn-recovery" }; }, + async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, + providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; }, + async close() { + if (scenario === "close_retry" && ++closeAttempts === 1) throw new Error("First shutdown acknowledgement lost"); + closed = true; + }, + accountingUsageEvent: read, + }; + const appended: PrpEvent[] = []; + const enrichmentFailures: string[] = []; + const prepared = preparedInput(); + const { qualificationStatus: _status, permissionPolicy: _policy, ...profile } = QUALIFIED_ACPX_PROFILES.hermes; + const outcome = await executeNativeSession({ + input: scenario === "legacy_provider" ? input : { ...prepared, schema: "paperclip.native-execution-input.v6", + session: { ...prepared.session, driverKind: "acpx_runtime" }, + provider: { kind: "acpx", agent: "hermes", model: "fixture-model", permissionMode: "approve-all", profile, connectionFingerprint: "1".repeat(64) } }, + requireSessionCloseBeforeReturn: true, keepSessionOpen: scenario === "warm", + backend: { async descriptor() { return { kind: "mock", name: "shutdown-usage", version: "1", capabilities: await session.capabilities(), + runtimeContextCapabilities: { instructions: "native", skills: "native", mcp: "native" } }; }, + async openSession() { return session; } }, + controlPlane: { async openRun() {}, async checkpointSession() {}, + async appendEvent(event) { appended.push(event); return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {} }, + runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", + onPostCompletionEnrichmentFailure: async ({ stage }) => { enrichmentFailures.push(stage); }, + }); + if (["bound", "close_retry"].includes(scenario)) expect(outcome.settledUsageEvent).toEqual(usageEvent); + else expect(outcome.settledUsageEvent).toBeUndefined(); + expect(appended.some(event => event.payload.kind === "usage")).toBe(false); + if (["warm", "legacy_provider"].includes(scenario)) expect(read).not.toHaveBeenCalled(); + else expect(read).toHaveBeenCalledOnce(); + if (!["bound", "close_retry", "warm", "legacy_provider"].includes(scenario)) expect(enrichmentFailures).toEqual(["usage"]); + if (scenario === "warm") await session.close({ reason: "test cleanup" }); + }, + ); + it.each([ { label: "continues a restart-lost Codex turn", allowed: true, error: { code: "provider_turn_lost_on_restore", recoverable: true } }, { label: "requires the control plane restart claim", allowed: false, error: { code: "provider_turn_lost_on_restore", recoverable: true } }, diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index 5003e54a5e8..58d3aabd25b 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -27,6 +27,7 @@ import { nativeRestartInterruptedTurnId, } from "./contracts/native-session-backend.js"; import { + validatePrpEvent, validatePrpStructuredRunResult, type PrpEvent, type PrpStructuredRunResult, @@ -2790,8 +2791,40 @@ export async function executeNativeSession( : descriptor.version, usage, }; + let settledUsageEvent: PrpEvent | undefined; + if (input.provider.kind === "acpx" && input.provider.agent === "hermes" + && options.requireSessionCloseBeforeReturn && !options.keepSessionOpen + && session.accountingUsageEvent) { + // Governed questions stop transcript consumption before Hermes emits + // terminal billing. Read the passive usage suffix only after this exact + // per-turn close proves shutdown. It cannot reopen task/tool authority. + void closeSession("Hermes per-turn accounting shutdown").catch(() => undefined); + await (sessionCloseRecoveryPromise ?? sessionClosePromise); + try { + const candidate = await runAbortableOperationWithin({ + timeoutMs: FAILED_OPERATION_SETTLEMENT_GRACE_MS, + timeoutMessage: "native shutdown accounting read timed out", + operation: () => session.accountingUsageEvent!(), + }); + const validation = candidate === null ? null : validatePrpEvent(candidate); + if (candidate !== null) { + if (!validation?.ok || candidate.sourceKind !== "runner" + || candidate.sourceInstanceId !== consumed.event?.sourceInstanceId + || candidate.runId !== identity.runId + || candidate.normalizedSessionId !== normalizedSessionId + || candidate.turnId !== durableExecutionResult.turnId + || candidate.eventType !== "item.completed" || candidate.payload.kind !== "usage") { + throw new Error("native shutdown accounting event binding mismatch"); + } + settledUsageEvent = structuredClone(candidate); + } + } catch (error) { + await observeEnrichmentFailure("usage", error); + } + } executionSucceeded = true; - return { ...durableExecutionResult, ...enrichment }; + return { ...durableExecutionResult, ...enrichment, + ...(settledUsageEvent ? { settledUsageEvent } : {}) }; } catch (error) { executionFailure = { error }; throw error; diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 2e36f4d1038..2f4957658cc 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -5508,6 +5508,48 @@ describe("native terminal-turn accounting", () => { } }); + it.each(["shutdown", "duplicate", "wrong_biller"] as const)("journals closed Hermes usage after governed input yield (%s)", async scenario => { + const previous = process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION; + process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION = JSON.stringify([{ agent: "hermes", model: "fixture-model" }]); + try { + const onUsage = vi.fn(async (_receipt: import("@paperclipai/adapter-utils").AdapterUsageCheckpoint) => {}); + const usage: PrpEvent = { + schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceInstanceId: "provider", + sourceEventId: "provider:shutdown-usage:2", sourceSeq: 2, priority: 1, + runId: execution.binding.runId, normalizedSessionId: "session", turnId: "turn", + eventType: "item.completed", emittedAt: new Date().toISOString(), + payload: { kind: "usage", usage: { + runDelta: { inputTokens: 31, outputTokens: 7, providerCostUsd: 0.25 }, runDeltaComplete: true, + billing: { schema: "paperclip.usage.billing/v1", source: "provider_reported", biller: "openrouter", currency: "USD", + complete: true, requestCount: 1, reportedRequestCount: 1, amountUsd: 0.25, amountUsdExact: "0.250000000" }, + } }, + }; + state.execute.mockReset().mockImplementationOnce(async () => { + await accountingEvents.committed!({ ...usage, sourceSeq: 1, sourceEventId: "provider:shutdown-usage:1", + eventType: "turn.started", payload: {} }); + if (scenario === "duplicate") await accountingEvents.committed!(usage); + return { result: { summary: "Waiting for an answer", reportedWorkDisposition: "yielded" }, + terminal: { runTerminalState: "succeeded", reportedWorkDisposition: "yielded" }, turnId: "turn", + normalizedSessionId: "session", providerSessionId: null, driverKind: "test", driverVersion: "1", + nativeEventCount: 1, highestContiguousSourceSeq: 1, + // An unrelated attachment/session snapshot must not supply billing. + usage: { runDelta: { inputTokens: 999, outputTokens: 999, providerCostUsd: 999 } }, + settledUsageEvent: usage }; + }); + const result = await executePaperclipNativeSession({ db: leaseDb(), runnerInstanceId: "runner", onUsage, + billingIdentity: { provider: "deepseek", biller: scenario === "wrong_biller" ? "anthropic" : "openrouter", billingType: "metered_api" }, + execution: { ...execution, provider: { kind: "acpx", agent: "hermes", model: "fixture-model", permissionMode: "approve-all", profile: QUALIFIED_ACPX_PROFILES.hermes } } as NativeExecutionInput }); + expect(result).toMatchObject({ usageComplete: true, costUsd: scenario === "wrong_biller" ? null : 0.25, + costStatus: scenario === "wrong_biller" ? "unpriced" : "reported", usage: { inputTokens: 31, outputTokens: 7 } }); + expect(onUsage.mock.calls.at(-1)![0]).toMatchObject({ costUsd: result.costUsd, complete: true }); + if (scenario === "wrong_biller") expect(result.settlement).toBeUndefined(); + else expect(result.settlement).toMatchObject({ providerWorkEnded: true, usageComplete: true }); + } finally { + if (previous === undefined) delete process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION; + else process.env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION = previous; + } + }); + it.each(["committed", "duplicate"] as const)("saves completed usage before returning to result commitment (%s)", async mode => { const onUsage = vi.fn(async (_receipt: import("@paperclipai/adapter-utils").AdapterUsageCheckpoint) => {}); const failure = new Error("stopped after result commit, before executor returns"); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index d8d45a9d839..140e5abc140 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -8786,6 +8786,20 @@ async function executePaperclipNativeSessionWithinScope( // session.usage() may be an attachment baseline, a partial report, or a // cumulative session total. Use the observed run delta, and require its // turn to match the terminal result before certifying it as complete. + if (input.execution.provider.kind === "acpx" && input.execution.provider.agent === "hermes" + && native.settledUsageEvent) { + // The runtime returns this fact only after the exact per-turn close. + // Persist through the existing durable receipt path; never infer price + // from a session usage baseline or reopen the terminated event consumer. + const event = native.settledUsageEvent; + if (!validatePrpEvent(event).ok || event.runId !== input.execution.binding.runId + || event.normalizedSessionId !== native.normalizedSessionId + || event.turnId !== native.turnId || event.eventType !== "item.completed" + || event.sourceKind !== "runner" || event.payload.kind !== "usage") { + throw new Error("native shutdown accounting event binding mismatch"); + } + await observeAccountingEvent(event); + } if (turnAccounting) { const snapshot = turnAccounting.finish(native.turnId); observedAccountingUsage = snapshot.usage; diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 65e9da495bc..2725f196cc4 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1478,6 +1478,13 @@ need their separate conformance/qualification evidence. Hermes uses an existing managed OpenRouter connection through the ordinary connection fixture. It remains pending qualification; the native transport fixture does not substitute for these browser and remote workflows. +The ten Hermes/OpenRouter cells set 200-cent company and agent budgets and +verify them through public reads before creating work. Before cleanup, every +run, including a run that yields for human input, must have settled reported +wire cost and healthy budget state. The per-run receipts are retained in +`hermes-openrouter-workflow-settlement.json`; `api-state.json` also retains all +selected runs. Extended-harnesses definition version 2 records this stronger +oracle. Older behavioral passes do not prove this accounting requirement. Before loading local credentials, the launcher records the checked-out controller SHA and ref for every selected Hermes candidate, including API and Bedrock cells. diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 0bb8a714a3a..abc5a7ae9c4 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1210,7 +1210,8 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ groups: ["native"], profiles: extendedHarnessProfiles, environments: runnerEnvironments, tasks: [...openRouterBreadthTasks, localIntegrityTasks[1]!, extendedHarnessFileTask], expectedMatrixSize: 40, - definitionMetadata: { version: 1, qualification: "pending", scheduling: "explicit-only", admission: "host-exact-candidate-and-model", authenticatedDiscoveryDate: "2026-09-28" }, + definitionMetadata: { version: 2, qualification: "pending", scheduling: "explicit-only", admission: "host-exact-candidate-and-model", authenticatedDiscoveryDate: "2026-09-28", + hermesBilling: "per-run-reported-cost-and-budget-health", hermesBudgetMonthlyCents: 200, hermesSettlementSourceDigest: hermesApiConnectionDefinitionDigest }, }, { id: "hermes-api-connections", label: "Hermes managed API connections", manualOnly: true, diff --git a/tests/runner-e2e/hermes-api-connections.test.ts b/tests/runner-e2e/hermes-api-connections.test.ts index a372425ce54..74a5f1f9619 100644 --- a/tests/runner-e2e/hermes-api-connections.test.ts +++ b/tests/runner-e2e/hermes-api-connections.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { runnerMatrix, runnerSuites, suiteDefinitionHash } from "./catalog.js"; import { buildMatrixJobs, parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; import { buildRunnerE2EProcessEnvironment } from "./harness-env.js"; -import { captureHermesApiAccountOwner, captureHermesApiBudgets, captureHermesOpenRouterSettlement, gradeHermesApiConnection } from "./hermes-api-connections.js"; +import { captureHermesApiAccountOwner, captureHermesApiBudgets, captureHermesOpenRouterSettlement, gradeHermesApiConnection, isHermesOpenRouterWorkflow } from "./hermes-api-connections.js"; describe("Hermes managed API connection qualification", () => { const suite = runnerSuites.find(s => s.id === "hermes-api-connections")!; @@ -15,6 +15,16 @@ describe("Hermes managed API connection qualification", () => { expect(suite.definitionMetadata).toMatchObject({ qualification: "pending", accountMethod: "api_key", accountMode: "responsible_user", coverage: "api-account-native-completion-only", budgetMonthlyCents: 200, maximumAttemptsPerCell: 1 }); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(e => e.suite.id === suite.id)).toBe(false); }); + it("requires accounting for every Hermes OpenRouter product workflow without widening other provider cells", () => { + const workflow = runnerMatrix.filter(isHermesOpenRouterWorkflow); + expect(workflow).toHaveLength(10); + expect(new Set(workflow.map(cell => cell.task.id))).toEqual(new Set([ + "hello-complete", "question-resume-complete", "plan-approve-complete", "structured-question-restart-resume", "file-edit-validate", + ])); + expect(workflow.every(cell => cell.suite.definitionMetadata?.hermesBudgetMonthlyCents === 200)).toBe(true); + expect(isHermesOpenRouterWorkflow({ ...workflow[0]!, profile: { ...workflow[0]!.profile, qualificationCandidate: "cursor" } })).toBe(false); + expect(isHermesOpenRouterWorkflow({ ...workflow[0]!, profile: { ...workflow[0]!.profile, credential: "ANTHROPIC_API_KEY" } })).toBe(false); + }); it.each([ ["XAI_API_KEY", "grok-4.7"], ["GEMINI_API_KEY", "gemini-3.8-flash"], diff --git a/tests/runner-e2e/hermes-api-connections.ts b/tests/runner-e2e/hermes-api-connections.ts index cd6a6f341ec..600fa8d35f0 100644 --- a/tests/runner-e2e/hermes-api-connections.ts +++ b/tests/runner-e2e/hermes-api-connections.ts @@ -10,6 +10,10 @@ const present = (value: unknown) => typeof value === "string" && value.trim().le export const HERMES_API_CONNECTION_BUDGET_CENTS = 200; export const isHermesConnectionSuite = (suiteId: string) => suiteId === "hermes-api-connections" || suiteId === "hermes-bedrock-connections"; +export const isHermesOpenRouterWorkflow = (execution: { + suite: { id: string }; profile: { qualificationCandidate?: string; credential: string }; +}) => execution.suite.id === "extended-harnesses" + && execution.profile.qualificationCandidate === "hermes" && execution.profile.credential === "OPENROUTER_API_KEY"; export const hermesBedrockConnectionChoice = { credential: "AWS_BEARER_TOKEN_BEDROCK", model: "us.anthropic.claude-haiku-4-5-20251001-v1:0", diff --git a/tests/runner-e2e/live-fixtures.test.ts b/tests/runner-e2e/live-fixtures.test.ts index 827d781cdd0..adde436386b 100644 --- a/tests/runner-e2e/live-fixtures.test.ts +++ b/tests/runner-e2e/live-fixtures.test.ts @@ -97,6 +97,7 @@ describe("live runner fixtures", () => { ["runner-acpx-hermes-api-openai", "hermes-api-connections", "hello-complete"], ["runner-acpx-hermes-api-xai", "hermes-api-connections", "hello-complete"], ["runner-acpx-hermes-api-google", "hermes-api-connections", "hello-complete"], + ["runner-acpx-hermes", "extended-harnesses", "question-resume-complete"], ["runner-acpx-hermes-bedrock", "hermes-bedrock-connections", "hello-complete"], ["runner-codex", "hiring-templates", "hire-coder-template-reuse"], ["runner-acpx-claude", "hiring-templates", "hire-coder-template-reuse"], @@ -161,7 +162,7 @@ describe("live runner fixtures", () => { }, }); expect(connected).toBe(true); - if (["hermes-api-connections", "hermes-bedrock-connections"].includes(suite)) { + if (["hermes-api-connections", "hermes-bedrock-connections"].includes(suite) || profile === "runner-acpx-hermes") { expect(companyBody.budgetMonthlyCents).toBe(200); expect(agentBody.budgetMonthlyCents).toBe(200); } diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index 0118e5bb455..fbad3840581 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -1,5 +1,5 @@ import { NATIVE_COMPLETION_BUDGET_CENTS } from "./native-completion-defaults.js"; -import { HERMES_API_CONNECTION_BUDGET_CENTS, isHermesConnectionSuite } from "./hermes-api-connections.js"; +import { HERMES_API_CONNECTION_BUDGET_CENTS, isHermesConnectionSuite, isHermesOpenRouterWorkflow } from "./hermes-api-connections.js"; import type { AiConnectionBinding } from "../../packages/shared/src/ai-connections.js"; import path from "node:path"; import { installedReleaseDaytonaPlugin } from "./installed-release.js"; @@ -137,7 +137,7 @@ export async function setupLiveFixtures(input: { return api.post("/api/companies", { name: `Runner E2E ${execution.id} ${input.executionNonce}`, description: "Ephemeral paid full-stack runner acceptance fixture", - budgetMonthlyCents: isHermesConnectionSuite(execution.suite.id) ? HERMES_API_CONNECTION_BUDGET_CENTS + budgetMonthlyCents: isHermesConnectionSuite(execution.suite.id) || isHermesOpenRouterWorkflow(execution) ? HERMES_API_CONNECTION_BUDGET_CENTS : ["native-completion", "native-instruction-consolidation", "native-connection-guidance"].includes(execution.suite.id) || (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id)) ? NATIVE_COMPLETION_BUDGET_CENTS : execution.suite.id === "task-titles" ? TASK_TITLE_BUDGET_CENTS @@ -306,7 +306,7 @@ export async function setupLiveFixtures(input: { || (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id))) { agent.budgetMonthlyCents = 1_000; } - if (isHermesConnectionSuite(execution.suite.id)) agent.budgetMonthlyCents = HERMES_API_CONNECTION_BUDGET_CENTS; + if (isHermesConnectionSuite(execution.suite.id) || isHermesOpenRouterWorkflow(execution)) agent.budgetMonthlyCents = HERMES_API_CONNECTION_BUDGET_CENTS; if (managedHiring) { const account = value(resolved, "ai-connection"); const config = agent.adapterConfig as Record; diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 64c2b780667..077fe460dae 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,5 +1,5 @@ import { runPlanTaskFlow } from "./plan-task-flow.js"; -import { captureHermesApiAccountOwner, captureHermesApiBudgets, captureHermesOpenRouterSettlement, gradeHermesApiConnection, isHermesConnectionSuite } from "./hermes-api-connections.js"; +import { captureHermesApiAccountOwner, captureHermesApiBudgets, captureHermesOpenRouterSettlement, gradeHermesApiConnection, isHermesConnectionSuite, isHermesOpenRouterWorkflow } from "./hermes-api-connections.js"; import { assertNativeCompletionSelection, NATIVE_COMPLETION_PREFLIGHT_ENV, verifyNativeCompletionPreflight } from "./native-completion-admission.js"; import { assertNativeInstructionSelection, verifyNativeInstructionPreflight, NATIVE_INSTRUCTION_PREFLIGHT_ENV, NATIVE_INSTRUCTION_SUITE, NATIVE_INSTRUCTION_DEFAULT_SHA256 } from "./native-instruction-consolidation.js"; import { captureNativeDefault, gradeNativeDefault, nativeCompletionWorkspaceDigest } from "./native-completion-defaults.js"; @@ -907,6 +907,11 @@ for (const execution of executions) { await writeSanitizedJson(snapshotsDir, "hermes-api-budgets-before-execution.json", receipt, secrets); if (!receipt.checks.every(check => check.passed)) throw new Error("Hermes API budgets failed admission before task creation"); } + if (isHermesOpenRouterWorkflow(execution)) { + const receipt = await captureHermesApiBudgets({ api, companyId: fixtures.company.id, agentId: fixtures.agent.id }); + await writeSanitizedJson(snapshotsDir, "hermes-workflow-budgets-before-execution.json", receipt, secrets); + if (!receipt.checks.every(check => check.passed)) throw new Error("Hermes workflow budgets failed admission before task creation"); + } if (["native-completion", NATIVE_INSTRUCTION_SUITE].includes(execution.suite.id)) { const receipt = await captureNativeDefault({ api, agentId: fixtures.agent.id, companyId: fixtures.company.id }); const grade = gradeNativeDefault(receipt, execution.suite.id === NATIVE_INSTRUCTION_SUITE ? NATIVE_INSTRUCTION_DEFAULT_SHA256 : undefined); @@ -2707,6 +2712,7 @@ for (const execution of executions) { { issue, run, + runs: selectedRuns, comments: terminal.comments, interactions: terminal.interactions, planLifecycleEvidence, @@ -2916,6 +2922,22 @@ for (const execution of executions) { await writeSanitizedJson(snapshotsDir, "hermes-api-connection.json", { checks }, secrets); expect(checks.every(check => check.passed), "Hermes managed account and native model attribution").toBe(true); } + if (isHermesOpenRouterWorkflow(execution)) { + let receipts: Array>> = []; + try { + await expect.poll(async () => { + receipts = await Promise.all(selectedRuns.map(run => captureHermesOpenRouterSettlement({ api, + companyId: fixtures!.company.id, agentId: fixtures!.agent.id, issueId: issue!.id, runId: run.id }))); + return receipts.length === selectedRuns.length && receipts.every(receipt => receipt.checks.every(check => check.passed)); + }, { timeout: 30_000, message: "Every Hermes workflow run must settle reported cost and retain budget health before cleanup" }).toBe(true); + } finally { + await writeSanitizedJson(snapshotsDir, "hermes-openrouter-workflow-settlement.json", { receipts }, secrets); + matcherResults.push(...receipts.flatMap((receipt, index) => receipt.checks.map(check => ({ + matcher: { kind: "json_path" as const, path: `hermesBilling.run${index + 1}.${check.id}`, expected: true }, passed: check.passed, + detail: "Each public run, including input-yield runs, must have settled reported cost and healthy budgets before teardown.", + })))); + } + } if (runsCompletionUpdateProbe(execution) && credentials.OPENAI_API_KEY) { const qualification = completionQualityStatus(completionQuality); if (qualification === "unqualified") {