diff --git a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts index aee05c5605b..f61f6a2137a 100644 --- a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts +++ b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts @@ -305,6 +305,33 @@ describe("eval-session usage", () => { })).toThrow("agent turn limit exceeded"); }); + it("counts prior unavailable candidate turns alongside a later measured turn without claiming a complete total", () => { + const parsed = parseEvalSessionRequest(request({ + provider: "acpx", acpxAgent: "cursor", model: "gpt-5.6-sol", + }), { candidateProfile: "cursor" }); + const unavailable = { + turnId: "prior-turn", attemptId: "attempt-1", agent: "cursor" as const, + reason: "provider_did_not_report_usage" as const, + tokenUsage: null, costNanodollars: null, observedAt: "2026-09-28T19:18:10.000Z", + }; + const snapshot = { + config: { provider: "acpx", acpxAgent: "cursor" }, + usageUnavailable: [unavailable, { ...unavailable }], + usageLedger: [{ + receiptId: "measured-receipt", turnId: "measured-turn", providerCalls: 1, + providerRequests: 1, inputTokens: 100, outputTokens: 10, + cachedInputTokens: 0, reasoningTokens: 0, costNanodollars: 0, + }], + } as unknown as CapabilityLiveSessionSnapshot; + const turn = { turnId: "measured-turn", status: "completed" as const, assistantText: "Done", snapshot }; + expect(() => boundedEvalSessionUsage(parsed, turn)).toThrow("agent turn limit exceeded"); + expect(boundedEvalSessionUsage({ ...parsed, limits: { ...parsed.limits, maxAgentTurns: 2 } }, turn)).toBeNull(); + expect(boundedEvalSessionUsage(parsed, { + ...turn, + snapshot: { ...snapshot, usageUnavailable: [{ ...unavailable, turnId: "measured-turn" }] }, + })).toMatchObject({ agentTurns: 1, inputTokens: 100, outputTokens: 10 }); + }); + it("deduplicates receipts and applies the versioned model price", () => { const receipt = { receiptId: "receipt-1", diff --git a/packages/paperclip-runner/src/cli/eval-session.ts b/packages/paperclip-runner/src/cli/eval-session.ts index 659166ad117..081bc59a876 100644 --- a/packages/paperclip-runner/src/cli/eval-session.ts +++ b/packages/paperclip-runner/src/cli/eval-session.ts @@ -250,19 +250,23 @@ export function boundedEvalSessionUsage( if (turn.status !== "completed") { return usageIfAvailable(request, turn.snapshot); } - const unavailable = request.provider === "acpx" + const candidate = request.provider === "acpx" && request.acpxAgent !== undefined && ["pi", "cursor", "copilot"].includes(request.acpxAgent) && turn.snapshot.config.provider === "acpx" - && turn.snapshot.config.acpxAgent === request.acpxAgent - && turn.snapshot.usageUnavailable?.some((entry) => entry.turnId === turn.turnId - && entry.agent === request.acpxAgent && entry.reason === "provider_did_not_report_usage"); - const usage = unavailable + && turn.snapshot.config.acpxAgent === request.acpxAgent; + const measuredTurns = new Set(turn.snapshot.usageLedger?.map((entry) => entry.turnId)); + const unavailableTurnIds = new Set(candidate + ? turn.snapshot.usageUnavailable?.filter((entry) => entry.agent === request.acpxAgent + && entry.reason === "provider_did_not_report_usage" + && entry.tokenUsage === null && entry.costNanodollars === null + && !measuredTurns.has(entry.turnId)).map((entry) => entry.turnId) + : []); + const unavailable = unavailableTurnIds.size > 0; + const usage = unavailable && (unavailableTurnIds.has(turn.turnId) || turn.snapshot.usageLedger?.length) ? usageIfAvailable(request, turn.snapshot) : evalSessionUsage(request.model, turn.snapshot); - const unavailableTurns = unavailable - ? new Set(turn.snapshot.usageUnavailable?.map((entry) => entry.turnId)).size - : 0; + const unavailableTurns = unavailableTurnIds.size; if ((usage?.agentTurns ?? 0) + unavailableTurns > request.limits.maxAgentTurns) { throw new Error("agent turn limit exceeded"); }