diff --git a/packages/paperclip-runner/docs/evals-integration.md b/packages/paperclip-runner/docs/evals-integration.md index 0876983450a..34c241aeb63 100644 --- a/packages/paperclip-runner/docs/evals-integration.md +++ b/packages/paperclip-runner/docs/evals-integration.md @@ -63,6 +63,21 @@ Evals owns conversion of this bundle into Evalbook ledger, scores, and reports. The App package does not implement an Evalbook environment loader, importer, grid, comparison, or report renderer. +## Direct eval cost provenance + +Codex App Server supplies token usage, not a provider USD receipt. Its direct +`eval-session` result therefore sets `providerReportedCostNanodollars` to null +and `providerReportedCostProvenance` to `unavailable`. The pinned token-price +estimate remains in `estimatedCostNanodollars` with `costCoverage: estimated`. +A missing receipt must not appear as a free request. + +`legacyLedgerCostNanodollars` retains the unverified ledger amount for Codex. +This amount and the token estimate still participate in the existing budget +stop. The ledger amount is not provider billing, and consumers must not add it +to the estimate or treat its zero placeholder as proof of zero spend. Raw +usage receipts remain in the snapshot. Older artifacts remain their original +measurements; this change does not reconcile invoices or reprice history. + ## Deterministic conformance `runHarnessDriverConformance` with `DeterministicHarnessDriver` covers, without diff --git a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts index 559cba217ad..6b12436ec21 100644 --- a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts +++ b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; import { createCapabilityFixtureState } from "../mock-core/capability-control-plane-types.js"; +import { sumAttemptCosts } from "../../scripts/runner-protocol-eval-metrics.mjs"; import { parseNativeRuntimeContext } from "../contracts/runtime-context.js"; import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js"; @@ -451,6 +452,39 @@ describe("eval-session usage", () => { }); }); + it.each([0, 6_000_000])("does not label Codex legacy ledger cost %d as a provider USD receipt", (legacyCost) => { + const snapshot = { + config: { provider: "codex" }, + usageLedger: [{ + receiptId: "codex-receipt", providerCalls: 1, providerRequests: 1, + inputTokens: 1_000, outputTokens: 100, cachedInputTokens: 400, + reasoningTokens: 50, costNanodollars: legacyCost, + }], + } as unknown as CapabilityLiveSessionSnapshot; + const usage = boundedEvalSessionUsage(parseEvalSessionRequest(request()), { + turnId: "codex-turn", status: "completed", assistantText: "Completed", snapshot, + }); + expect(usage).toMatchObject({ + agentTurns: 1, providerRequests: 1, inputTokens: 1_000, outputTokens: 100, + providerReportedCostNanodollars: null, + providerReportedCostProvenance: "unavailable", + estimatedCostNanodollars: 6_200_000, + legacyLedgerCostNanodollars: legacyCost, + costCoverage: "estimated", + }); + expect(sumAttemptCosts([usage])).toMatchObject({ + providerReported: { nanodollars: null, recordedAttempts: 0 }, + estimated: { nanodollars: 6_200_000, recordedAttempts: 1 }, + }); + expect(snapshot.usageLedger[0]?.costNanodollars).toBe(legacyCost); + const limitedRequest = parseEvalSessionRequest(request()); + limitedRequest.limits.maxEstimatedCostNanodollars = 10_000_000; + const costlySnapshot = { ...snapshot, usageLedger: [{ ...snapshot.usageLedger[0]!, costNanodollars: 10_000_001 }] }; + expect(() => boundedEvalSessionUsage(limitedRequest, { + turnId: "codex-turn", status: "completed", assistantText: "Completed", snapshot: costlySnapshot, + })).toThrow("legacy ledger cost limit exceeded"); + }); + it("fails closed when a completed turn has no usage receipt", () => { expect(() => evalSessionUsage( "gpt-5.6-sol", diff --git a/packages/paperclip-runner/src/cli/eval-session-contract.ts b/packages/paperclip-runner/src/cli/eval-session-contract.ts index d134af83084..36a5a7dea6e 100644 --- a/packages/paperclip-runner/src/cli/eval-session-contract.ts +++ b/packages/paperclip-runner/src/cli/eval-session-contract.ts @@ -94,6 +94,7 @@ export interface EvalSessionUsage { cachedInputTokens: number; reasoningTokens: number; providerReportedCostNanodollars: number | null; + legacyLedgerCostNanodollars?: number; providerReportedCostProvenance: "legacy_runner_usage_ledger" | "unavailable"; estimatedCostNanodollars: number | null; pricingVersion: EstimatedModelCost["pricingVersion"] | null; @@ -401,6 +402,8 @@ export function evalSessionUsage( } const candidate = snapshot.config?.provider === "acpx" && ["pi", "cursor", "copilot"].includes(snapshot.config.acpxAgent ?? ""); + const codex = snapshot.config?.provider === "codex"; + const providerCostUnavailable = candidate || codex; let estimate: EstimatedModelCost | null; try { estimate = estimateModelCostNanodollars(model, totals); @@ -413,14 +416,12 @@ export function evalSessionUsage( } return { ...totals, - // Candidate ACP adapters do not supply authenticated USD receipts. The - // shared legacy ledger fills absent cost with zero, so its numeric value - // cannot establish an invoice amount for these profiles. - providerReportedCostNanodollars: candidate ? null : totals.providerReportedCostNanodollars, - providerReportedCostProvenance: candidate ? "unavailable" : "legacy_runner_usage_ledger", + ...(codex ? { legacyLedgerCostNanodollars: totals.providerReportedCostNanodollars } : {}), + providerReportedCostNanodollars: providerCostUnavailable ? null : totals.providerReportedCostNanodollars, + providerReportedCostProvenance: providerCostUnavailable ? "unavailable" : "legacy_runner_usage_ledger", estimatedCostNanodollars: estimate?.estimatedCostNanodollars ?? null, pricingVersion: estimate?.pricingVersion ?? null, ratesUsdPerMillionTokens: estimate?.ratesUsdPerMillionTokens ?? null, - costCoverage: candidate ? (estimate === null ? "unpriced" : "estimated") : "legacy_ledger_and_estimate", + costCoverage: providerCostUnavailable ? (estimate === null ? "unpriced" : "estimated") : "legacy_ledger_and_estimate", }; } diff --git a/packages/paperclip-runner/src/cli/eval-session.ts b/packages/paperclip-runner/src/cli/eval-session.ts index 0c012db55f0..788dc61602b 100644 --- a/packages/paperclip-runner/src/cli/eval-session.ts +++ b/packages/paperclip-runner/src/cli/eval-session.ts @@ -340,6 +340,9 @@ export function boundedEvalSessionUsage( ) { throw new EvalSessionBudgetError("provider-reported cost limit exceeded"); } + if (usage?.legacyLedgerCostNanodollars !== undefined && usage.legacyLedgerCostNanodollars > request.limits.maxEstimatedCostNanodollars) { + throw new EvalSessionBudgetError("legacy ledger cost limit exceeded"); + } if (unavailable || usage === null || (usage.estimatedCostNanodollars === null && usage.providerReportedCostNanodollars === null)) { throw new EvalSessionBudgetError("budget cost coverage is unavailable for one or more completed turns", true); } diff --git a/server/src/__tests__/muse-control.test.ts b/server/src/__tests__/muse-control.test.ts index f59ebdf7c78..c93fbf5b19e 100644 --- a/server/src/__tests__/muse-control.test.ts +++ b/server/src/__tests__/muse-control.test.ts @@ -55,7 +55,7 @@ describe("personal Muse control plane",()=>{ afterAll(async()=>{vi.unstubAllEnvs();await museReceiver(db).stop();if(externalDatabaseUrl)await closeRegisteredClients(externalDatabaseUrl);await temporary?.cleanup();}); async function fixture(ready=true) { const operatorId=randomUUID();await db.insert(authUsers).values({id:operatorId,name:"Muse authorizer",email:operatorId+"@example.test",createdAt:new Date(),updatedAt:new Date()}); - const [company]=await db.insert(companies).values({name:"Muse control",issuePrefix:"MU"+randomBytes(3).toString("hex")}).returning(); + const [company]=await db.insert(companies).values({name:"Muse control",issuePrefix:"MU"+operatorId.replaceAll("-","")}).returning(); const [agent]=await db.insert(agents).values({companyId:company.id,name:"Muse",role:"engineer",status:"active",adapterType:"paperclip_runner",adapterConfig:{provider:"muse",allowUnmeteredProvider:true,lifecycle:"per_turn"},permissions:{canCreateTasks:true}}).returning(); await db.insert(companyMemberships).values([{companyId:company.id,principalType:"user",principalId:operatorId,membershipRole:"owner",status:"active"},{companyId:company.id,principalType:"agent",principalId:agent.id,membershipRole:"member",status:"active"}]); const ticket=randomBytes(32).toString("base64url");const [binding]=await db.insert(bindings).values({companyId:company.id,agentId:agent.id,operatorId,ticketHash:museCredentialHash(ticket),ticketExpiresAt:new Date(Date.now()+600000)}).returning();