From c41d134f72ed503d3603b674ff9a3c83ad66278e Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 28 Sep 2026 14:24:12 -0500 Subject: [PATCH] fix(runner): retain candidate results without invented usage receipts Record explicit unavailable usage for opted-in ACP candidates and preserve their semantic oracle result without creating zero-token or zero-cost receipts. Qualified providers retain strict accounting; unknown candidate turns still count toward the turn limit. Co-Authored-By: Paperclip --- .../src/cli/eval-session-contract.test.ts | 27 +++++++++ .../paperclip-runner/src/cli/eval-session.ts | 22 +++++-- .../src/live/live-session.test.ts | 60 ++++++++++++++++++- .../paperclip-runner/src/live/live-session.ts | 60 +++++++++++++++++-- 4 files changed, 157 insertions(+), 12 deletions(-) diff --git a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts index e616612f198..aee05c5605b 100644 --- a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts +++ b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts @@ -278,6 +278,33 @@ describe("eval-session usage", () => { })).toThrow("agent turn limit exceeded"); }); + it.each(["pi", "cursor", "copilot"] as const)("keeps %s oracle results when the current turn explicitly has unavailable usage", (agent) => { + const parsed = parseEvalSessionRequest(request({ + provider: "acpx", acpxAgent: agent, model: "exact-provider-model", + }), { candidateProfile: agent }); + const unavailable = { + turnId: "turn-candidate", attemptId: "attempt-1", agent, + reason: "provider_did_not_report_usage" as const, + tokenUsage: null, costNanodollars: null, observedAt: "2026-09-28T19:18:10.000Z", + }; + const snapshot = { + config: { provider: "acpx", acpxAgent: agent }, + usageLedger: [], usageUnavailable: [unavailable], + } as unknown as CapabilityLiveSessionSnapshot; + const turn = { turnId: unavailable.turnId, status: "completed" as const, assistantText: "Upgrade your plan to continue", snapshot }; + expect(boundedEvalSessionUsage(parsed, turn)).toBeNull(); + expect(turn.assistantText).toBe("Upgrade your plan to continue"); + expect(snapshot.usageLedger).toEqual([]); + expect(() => boundedEvalSessionUsage(parsed, { ...turn, turnId: "other-turn" })) + .toThrow("omitted usage accounting"); + expect(() => boundedEvalSessionUsage(parsed, { + ...turn, snapshot: { ...snapshot, usageUnavailable: [{ ...unavailable, agent: agent === "pi" ? "cursor" : "pi" }] }, + })).toThrow("omitted usage accounting"); + expect(() => boundedEvalSessionUsage(parsed, { + ...turn, snapshot: { ...snapshot, usageUnavailable: [unavailable, { ...unavailable, turnId: "second-turn" }] }, + })).toThrow("agent turn limit exceeded"); + }); + it("deduplicates receipts and applies the versioned model price", () => { const receipt = { receiptId: "receipt-1", diff --git a/packages/paperclip-runner/src/cli/eval-session.ts b/packages/paperclip-runner/src/cli/eval-session.ts index 9eee120d765..659166ad117 100644 --- a/packages/paperclip-runner/src/cli/eval-session.ts +++ b/packages/paperclip-runner/src/cli/eval-session.ts @@ -250,25 +250,37 @@ export function boundedEvalSessionUsage( if (turn.status !== "completed") { return usageIfAvailable(request, turn.snapshot); } - const usage = evalSessionUsage(request.model, turn.snapshot); - if (usage.agentTurns > request.limits.maxAgentTurns) { + const unavailable = request.provider === "acpx" + && request.acpxAgent !== undefined + && ["pi", "cursor", "copilot"].includes(request.acpxAgent) + && turn.snapshot.config.provider === "acpx" + && turn.snapshot.config.acpxAgent === request.acpxAgent + && turn.snapshot.usageUnavailable?.some((entry) => entry.turnId === turn.turnId + && entry.agent === request.acpxAgent && entry.reason === "provider_did_not_report_usage"); + const usage = unavailable + ? usageIfAvailable(request, turn.snapshot) + : evalSessionUsage(request.model, turn.snapshot); + const unavailableTurns = unavailable + ? new Set(turn.snapshot.usageUnavailable?.map((entry) => entry.turnId)).size + : 0; + if ((usage?.agentTurns ?? 0) + unavailableTurns > request.limits.maxAgentTurns) { throw new Error("agent turn limit exceeded"); } if ( - usage.estimatedCostNanodollars !== null && + usage !== null && usage.estimatedCostNanodollars !== null && usage.estimatedCostNanodollars > request.limits.maxEstimatedCostNanodollars ) { throw new Error("estimated cost limit exceeded"); } if ( - usage.providerReportedCostNanodollars !== null && + usage !== null && usage.providerReportedCostNanodollars !== null && usage.providerReportedCostNanodollars > request.limits.maxEstimatedCostNanodollars ) { throw new Error("provider-reported cost limit exceeded"); } - return usage; + return unavailable ? null : usage; } async function closeSession( diff --git a/packages/paperclip-runner/src/live/live-session.test.ts b/packages/paperclip-runner/src/live/live-session.test.ts index 9b250f1fec7..7f61d72a55f 100644 --- a/packages/paperclip-runner/src/live/live-session.test.ts +++ b/packages/paperclip-runner/src/live/live-session.test.ts @@ -75,6 +75,8 @@ interface FakeProviderState { holdAfterTool: boolean; closeError: Error | null; usageRunDelta: Record | null; + omitReadUsage?: boolean; + assistantReply?: string; onTurnStart?: () => Promise; onUsage?: (queue: AsyncNotifications, turnId: string) => void | Promise; /** Delays the fake `turn/interrupt` reply, to model a slow transport round trip. */ @@ -116,14 +118,14 @@ class FakeCapabilityCodexTransport implements CodexAppServerTransport { id: this.state.threadId, sessionId: this.state.providerSessionId, turns: [...this.state.turns].map(([id, status]) => ({ id, status })), - tokenUsage: { + ...(this.state.omitReadUsage ? {} : { tokenUsage: { total: { inputTokens: this.state.nextTurn * 100, cachedInputTokens: this.state.nextTurn * 20, outputTokens: this.state.nextTurn * 10, reasoningOutputTokens: this.state.nextTurn * 2, }, - }, + } }), }, }; } @@ -260,7 +262,7 @@ class FakeCapabilityCodexTransport implements CodexAppServerTransport { params: { threadId: this.state.threadId, turnId, - item: { id: `message-${turnId}`, type: "agentMessage", text: assistantText }, + item: { id: `message-${turnId}`, type: "agentMessage", text: this.state.assistantReply ?? assistantText }, }, }); this.state.turns.set(turnId, "completed"); @@ -1077,6 +1079,58 @@ describe("Capability live runnerd and Codex session", () => { await service.shutdown(session.id); }); + it.each(["pi", "cursor", "copilot"] as const)("retains a completed %s result with explicit unavailable usage and no fabricated receipt", async (acpxAgent) => { + const state = providerState(); + state.omitReadUsage = true; + state.assistantReply = "Upgrade your plan to continue"; + state.onUsage = (queue, turnId) => queue.push({ + method: "turn/completed", + params: { threadId: state.threadId, turn: { id: turnId, status: "completed" } }, + }); + const store = new InMemoryCapabilityLiveSessionStore(); + const service = new CapabilityLiveSessionService({ + store, + transportFactory: fakeTransportFactory(state), + transportOptions: { acpxCandidateProfile: acpxAgent }, + }); + const session = await service.create({ + provider: "acpx", acpxAgent, + requestedModel: acpxAgent === "pi" ? "openrouter/deepseek/deepseek-v4-flash-0731" : "exact-model", + }); + const result = await session.sendMessage("Orient to this task."); + expect(result.status).toBe("completed"); + expect(result.assistantText).toBe(state.assistantReply); + expect(result.snapshot.usageLedger).toEqual([]); + expect(result.snapshot.usageUnavailable).toEqual([{ + turnId: result.turnId, + attemptId: result.snapshot.currentAttemptId, + agent: acpxAgent, + reason: "provider_did_not_report_usage", + tokenUsage: null, + costNanodollars: null, + observedAt: expect.any(String), + }]); + expect(result.snapshot.authorizationRecords.filter((entry) => entry.phase !== "exposure")).toEqual([]); + await service.shutdown(session.id); + expect((await store.load(session.id))?.usageUnavailable).toEqual(result.snapshot.usageUnavailable); + expect((await store.load(session.id))?.process?.runnerExited).toBe(true); + }); + + it("still rejects missing usage for a qualified provider", async () => { + const state = providerState(); + state.omitReadUsage = true; + state.onUsage = (queue, turnId) => queue.push({ + method: "turn/completed", + params: { threadId: state.threadId, turn: { id: turnId, status: "completed" } }, + }); + const service = new CapabilityLiveSessionService({ transportFactory: fakeTransportFactory(state) }); + const session = await service.create(); + await expect(session.sendMessage("Orient to this task.")).rejects.toThrow("capability_live_usage_missing"); + expect(session.snapshot().usageLedger).toEqual([]); + expect(session.snapshot().usageUnavailable).toBeUndefined(); + await service.shutdown(session.id); + }); + it("streams normalized usage before terminal with cumulative fallback and explicit run deltas", async () => { const state = providerState(); const service = new CapabilityLiveSessionService({ transportFactory: fakeTransportFactory(state) }); diff --git a/packages/paperclip-runner/src/live/live-session.ts b/packages/paperclip-runner/src/live/live-session.ts index 82ff2f21cc0..b7827b6bb5d 100644 --- a/packages/paperclip-runner/src/live/live-session.ts +++ b/packages/paperclip-runner/src/live/live-session.ts @@ -199,6 +199,16 @@ export interface CapabilityLiveUsageReceipt { observedAt: string; } +export interface CapabilityLiveUnavailableUsage { + turnId: string; + attemptId: string; + agent: "pi" | "cursor" | "copilot"; + reason: "provider_did_not_report_usage"; + tokenUsage: null; + costNanodollars: null; + observedAt: string; +} + export interface CapabilityLiveStateRevision { revision: number; at: string; @@ -266,6 +276,8 @@ export interface CapabilityLiveSessionSnapshot { terminalTurns?: CapabilityLiveTurnTerminalFact[]; /** Exact-once provider usage/cost receipts retained across every attempt. */ usageLedger?: CapabilityLiveUsageReceipt[]; + /** Candidate turns without a provider receipt; these are never zero-valued receipts. */ + usageUnavailable?: CapabilityLiveUnavailableUsage[]; /** Process-level evidence for an interrupted Codex semantic-call resume. */ nativeResume?: { schema: "paperclip.runner.native-resume-proof/v1"; @@ -768,6 +780,24 @@ export function assertCapabilityLiveSessionSnapshot( } usageReceiptIds.add(receiptId); } + if (snapshot.usageUnavailable !== undefined && !Array.isArray(snapshot.usageUnavailable)) { + throw new Error("capability_live_checkpoint_corrupt: invalid unavailable usage"); + } + const unavailableTurnIds = new Set(); + for (const value of snapshot.usageUnavailable ?? []) { + const unavailable = record(value); + const turnId = text(unavailable.turnId); + if (config.provider !== "acpx" || unavailable.agent !== config.acpxAgent + || !["pi", "cursor", "copilot"].includes(text(unavailable.agent)) + || !turnId || turnId.length > 512 || unavailableTurnIds.has(turnId) + || !text(unavailable.attemptId) || text(unavailable.attemptId).length > 512 + || !text(unavailable.observedAt) + || unavailable.reason !== "provider_did_not_report_usage" + || unavailable.tokenUsage !== null || unavailable.costNanodollars !== null) { + throw new Error("capability_live_checkpoint_corrupt: invalid unavailable usage"); + } + unavailableTurnIds.add(turnId); + } if ( snapshot.stateHistory !== undefined && (!Array.isArray(snapshot.stateHistory) || snapshot.stateHistory.length > 256) @@ -1147,6 +1177,7 @@ export class CapabilityLiveSession { readonly #attempts: CapabilityLiveAttemptSnapshot[]; readonly #terminalTurns: CapabilityLiveTurnTerminalFact[]; readonly #usageLedger: CapabilityLiveUsageReceipt[]; + readonly #usageUnavailable: CapabilityLiveUnavailableUsage[]; readonly #stateHistory: CapabilityLiveStateRevision[]; readonly #workspaceDiffs: CapabilityLiveWorkspaceDiffEntry[]; readonly #workspaceFileReferences: CapabilityLiveWorkspaceFileReferenceEntry[]; @@ -1204,6 +1235,7 @@ export class CapabilityLiveSession { this.#currentAttemptId = initialAttemptId; this.#terminalTurns = structuredClone(options.snapshot?.terminalTurns ?? []); this.#usageLedger = structuredClone(options.snapshot?.usageLedger ?? []); + this.#usageUnavailable = structuredClone(options.snapshot?.usageUnavailable ?? []); this.#stateHistory = structuredClone(options.snapshot?.stateHistory ?? [{ revision: this.#port.snapshot().revision, at: this.#createdAt, @@ -1389,6 +1421,7 @@ export class CapabilityLiveSession { currentAttemptId: this.#currentAttemptId, terminalTurns: structuredClone(this.#terminalTurns), usageLedger: structuredClone(this.#usageLedger), + ...(this.#usageUnavailable.length === 0 ? {} : { usageUnavailable: structuredClone(this.#usageUnavailable) }), stateHistory: structuredClone(this.#stateHistory), workspaceDiffs: structuredClone(this.#workspaceDiffs), workspaceFileReferences: structuredClone(this.#workspaceFileReferences), @@ -1772,10 +1805,29 @@ export class CapabilityLiveSession { ...selected, costNanodollars: Math.max(selected.costNanodollars, reported?.costNanodollars ?? 0), }; - if ( - (selectedWithReportedCost?.inputTokens ?? 0) + (selectedWithReportedCost?.outputTokens ?? 0) === 0 - && !allowEmpty - ) { + const missingTokens = (selectedWithReportedCost?.inputTokens ?? 0) + + (selectedWithReportedCost?.outputTokens ?? 0) === 0; + const candidate = this.#config.acpxAgent; + if (missingTokens && this.#config.provider === "acpx" + && (candidate === "pi" || candidate === "cursor" || candidate === "copilot") + && this.#transportOptions.acpxCandidateProfile === candidate) { + // Native candidate wrappers can complete a turn without a usage receipt, + // including entitlement-denied turns. Retain the actual result for the + // oracle without inventing tokens, charges, or a successful model call. + if (!this.#usageUnavailable.some((entry) => entry.turnId === turnId)) { + this.#usageUnavailable.push({ + turnId, + attemptId: this.#currentAttemptId, + agent: candidate, + reason: "provider_did_not_report_usage", + tokenUsage: null, + costNanodollars: null, + observedAt: this.#now().toISOString(), + }); + } + return; + } + if (missingTokens && !allowEmpty) { throw new Error(`capability_live_usage_missing:${JSON.stringify({ captured: captured !== undefined, needsRead,