fix(runner): retain candidate results without invented usage receipts

Record explicit unavailable usage for opted-in ACP candidates and preserve their semantic oracle result without creating zero-token or zero-cost receipts. Qualified providers retain strict accounting; unknown candidate turns still count toward the turn limit.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-09-28 14:24:12 -05:00
1 parent af06ada99f
commit c41d134f72
4 files changed
+157 -12

No files matched your search

@@ -278,6 +278,33 @@ describe("eval-session usage", () => {
})).toThrow("agent turn limit exceeded");
});
it.each(["pi", "cursor", "copilot"] as const)("keeps %s oracle results when the current turn explicitly has unavailable usage", (agent) => {
const parsed = parseEvalSessionRequest(request({
provider: "acpx", acpxAgent: agent, model: "exact-provider-model",
}), { candidateProfile: agent });
const unavailable = {
turnId: "turn-candidate", attemptId: "attempt-1", agent,
reason: "provider_did_not_report_usage" as const,
tokenUsage: null, costNanodollars: null, observedAt: "2026-09-28T19:18:10.000Z",
};
const snapshot = {
config: { provider: "acpx", acpxAgent: agent },
usageLedger: [], usageUnavailable: [unavailable],
} as unknown as CapabilityLiveSessionSnapshot;
const turn = { turnId: unavailable.turnId, status: "completed" as const, assistantText: "Upgrade your plan to continue", snapshot };
expect(boundedEvalSessionUsage(parsed, turn)).toBeNull();
expect(turn.assistantText).toBe("Upgrade your plan to continue");
expect(snapshot.usageLedger).toEqual([]);
expect(() => boundedEvalSessionUsage(parsed, { ...turn, turnId: "other-turn" }))
.toThrow("omitted usage accounting");
expect(() => boundedEvalSessionUsage(parsed, {
...turn, snapshot: { ...snapshot, usageUnavailable: [{ ...unavailable, agent: agent === "pi" ? "cursor" : "pi" }] },
})).toThrow("omitted usage accounting");
expect(() => boundedEvalSessionUsage(parsed, {
...turn, snapshot: { ...snapshot, usageUnavailable: [unavailable, { ...unavailable, turnId: "second-turn" }] },
})).toThrow("agent turn limit exceeded");
});
it("deduplicates receipts and applies the versioned model price", () => {
const receipt = {
receiptId: "receipt-1",
@@ -250,25 +250,37 @@ export function boundedEvalSessionUsage(
if (turn.status !== "completed") {
return usageIfAvailable(request, turn.snapshot);
}
const usage = evalSessionUsage(request.model, turn.snapshot);
if (usage.agentTurns > request.limits.maxAgentTurns) {
const unavailable = request.provider === "acpx"
&& request.acpxAgent !== undefined
&& ["pi", "cursor", "copilot"].includes(request.acpxAgent)
&& turn.snapshot.config.provider === "acpx"
&& turn.snapshot.config.acpxAgent === request.acpxAgent
&& turn.snapshot.usageUnavailable?.some((entry) => entry.turnId === turn.turnId
&& entry.agent === request.acpxAgent && entry.reason === "provider_did_not_report_usage");
const usage = unavailable
? usageIfAvailable(request, turn.snapshot)
: evalSessionUsage(request.model, turn.snapshot);
const unavailableTurns = unavailable
? new Set(turn.snapshot.usageUnavailable?.map((entry) => entry.turnId)).size
: 0;
if ((usage?.agentTurns ?? 0) + unavailableTurns > request.limits.maxAgentTurns) {
throw new Error("agent turn limit exceeded");
}
if (
usage.estimatedCostNanodollars !== null &&
usage !== null && usage.estimatedCostNanodollars !== null &&
usage.estimatedCostNanodollars >
request.limits.maxEstimatedCostNanodollars
) {
throw new Error("estimated cost limit exceeded");
}
if (
usage.providerReportedCostNanodollars !== null &&
usage !== null && usage.providerReportedCostNanodollars !== null &&
usage.providerReportedCostNanodollars >
request.limits.maxEstimatedCostNanodollars
) {
throw new Error("provider-reported cost limit exceeded");
}
return usage;
return unavailable ? null : usage;
}
async function closeSession(
@@ -75,6 +75,8 @@ interface FakeProviderState {
holdAfterTool: boolean;
closeError: Error | null;
usageRunDelta: Record<string, unknown> | null;
omitReadUsage?: boolean;
assistantReply?: string;
onTurnStart?: () => Promise<void>;
onUsage?: (queue: AsyncNotifications, turnId: string) => void | Promise<void>;
/** Delays the fake `turn/interrupt` reply, to model a slow transport round trip. */
@@ -116,14 +118,14 @@ class FakeCapabilityCodexTransport implements CodexAppServerTransport {
id: this.state.threadId,
sessionId: this.state.providerSessionId,
turns: [...this.state.turns].map(([id, status]) => ({ id, status })),
tokenUsage: {
...(this.state.omitReadUsage ? {} : { tokenUsage: {
total: {
inputTokens: this.state.nextTurn * 100,
cachedInputTokens: this.state.nextTurn * 20,
outputTokens: this.state.nextTurn * 10,
reasoningOutputTokens: this.state.nextTurn * 2,
},
},
} }),
},
};
}
@@ -260,7 +262,7 @@ class FakeCapabilityCodexTransport implements CodexAppServerTransport {
params: {
threadId: this.state.threadId,
turnId,
item: { id: `message-${turnId}`, type: "agentMessage", text: assistantText },
item: { id: `message-${turnId}`, type: "agentMessage", text: this.state.assistantReply ?? assistantText },
},
});
this.state.turns.set(turnId, "completed");
@@ -1077,6 +1079,58 @@ describe("Capability live runnerd and Codex session", () => {
await service.shutdown(session.id);
});
it.each(["pi", "cursor", "copilot"] as const)("retains a completed %s result with explicit unavailable usage and no fabricated receipt", async (acpxAgent) => {
const state = providerState();
state.omitReadUsage = true;
state.assistantReply = "Upgrade your plan to continue";
state.onUsage = (queue, turnId) => queue.push({
method: "turn/completed",
params: { threadId: state.threadId, turn: { id: turnId, status: "completed" } },
});
const store = new InMemoryCapabilityLiveSessionStore();
const service = new CapabilityLiveSessionService({
store,
transportFactory: fakeTransportFactory(state),
transportOptions: { acpxCandidateProfile: acpxAgent },
});
const session = await service.create({
provider: "acpx", acpxAgent,
requestedModel: acpxAgent === "pi" ? "openrouter/deepseek/deepseek-v4-flash-0731" : "exact-model",
});
const result = await session.sendMessage("Orient to this task.");
expect(result.status).toBe("completed");
expect(result.assistantText).toBe(state.assistantReply);
expect(result.snapshot.usageLedger).toEqual([]);
expect(result.snapshot.usageUnavailable).toEqual([{
turnId: result.turnId,
attemptId: result.snapshot.currentAttemptId,
agent: acpxAgent,
reason: "provider_did_not_report_usage",
tokenUsage: null,
costNanodollars: null,
observedAt: expect.any(String),
}]);
expect(result.snapshot.authorizationRecords.filter((entry) => entry.phase !== "exposure")).toEqual([]);
await service.shutdown(session.id);
expect((await store.load(session.id))?.usageUnavailable).toEqual(result.snapshot.usageUnavailable);
expect((await store.load(session.id))?.process?.runnerExited).toBe(true);
});
it("still rejects missing usage for a qualified provider", async () => {
const state = providerState();
state.omitReadUsage = true;
state.onUsage = (queue, turnId) => queue.push({
method: "turn/completed",
params: { threadId: state.threadId, turn: { id: turnId, status: "completed" } },
});
const service = new CapabilityLiveSessionService({ transportFactory: fakeTransportFactory(state) });
const session = await service.create();
await expect(session.sendMessage("Orient to this task.")).rejects.toThrow("capability_live_usage_missing");
expect(session.snapshot().usageLedger).toEqual([]);
expect(session.snapshot().usageUnavailable).toBeUndefined();
await service.shutdown(session.id);
});
it("streams normalized usage before terminal with cumulative fallback and explicit run deltas", async () => {
const state = providerState();
const service = new CapabilityLiveSessionService({ transportFactory: fakeTransportFactory(state) });
@@ -199,6 +199,16 @@ export interface CapabilityLiveUsageReceipt {
observedAt: string;
}
export interface CapabilityLiveUnavailableUsage {
turnId: string;
attemptId: string;
agent: "pi" | "cursor" | "copilot";
reason: "provider_did_not_report_usage";
tokenUsage: null;
costNanodollars: null;
observedAt: string;
}
export interface CapabilityLiveStateRevision {
revision: number;
at: string;
@@ -266,6 +276,8 @@ export interface CapabilityLiveSessionSnapshot {
terminalTurns?: CapabilityLiveTurnTerminalFact[];
/** Exact-once provider usage/cost receipts retained across every attempt. */
usageLedger?: CapabilityLiveUsageReceipt[];
/** Candidate turns without a provider receipt; these are never zero-valued receipts. */
usageUnavailable?: CapabilityLiveUnavailableUsage[];
/** Process-level evidence for an interrupted Codex semantic-call resume. */
nativeResume?: {
schema: "paperclip.runner.native-resume-proof/v1";
@@ -768,6 +780,24 @@ export function assertCapabilityLiveSessionSnapshot(
}
usageReceiptIds.add(receiptId);
}
if (snapshot.usageUnavailable !== undefined && !Array.isArray(snapshot.usageUnavailable)) {
throw new Error("capability_live_checkpoint_corrupt: invalid unavailable usage");
}
const unavailableTurnIds = new Set<string>();
for (const value of snapshot.usageUnavailable ?? []) {
const unavailable = record(value);
const turnId = text(unavailable.turnId);
if (config.provider !== "acpx" || unavailable.agent !== config.acpxAgent
|| !["pi", "cursor", "copilot"].includes(text(unavailable.agent))
|| !turnId || turnId.length > 512 || unavailableTurnIds.has(turnId)
|| !text(unavailable.attemptId) || text(unavailable.attemptId).length > 512
|| !text(unavailable.observedAt)
|| unavailable.reason !== "provider_did_not_report_usage"
|| unavailable.tokenUsage !== null || unavailable.costNanodollars !== null) {
throw new Error("capability_live_checkpoint_corrupt: invalid unavailable usage");
}
unavailableTurnIds.add(turnId);
}
if (
snapshot.stateHistory !== undefined &&
(!Array.isArray(snapshot.stateHistory) || snapshot.stateHistory.length > 256)
@@ -1147,6 +1177,7 @@ export class CapabilityLiveSession {
readonly #attempts: CapabilityLiveAttemptSnapshot[];
readonly #terminalTurns: CapabilityLiveTurnTerminalFact[];
readonly #usageLedger: CapabilityLiveUsageReceipt[];
readonly #usageUnavailable: CapabilityLiveUnavailableUsage[];
readonly #stateHistory: CapabilityLiveStateRevision[];
readonly #workspaceDiffs: CapabilityLiveWorkspaceDiffEntry[];
readonly #workspaceFileReferences: CapabilityLiveWorkspaceFileReferenceEntry[];
@@ -1204,6 +1235,7 @@ export class CapabilityLiveSession {
this.#currentAttemptId = initialAttemptId;
this.#terminalTurns = structuredClone(options.snapshot?.terminalTurns ?? []);
this.#usageLedger = structuredClone(options.snapshot?.usageLedger ?? []);
this.#usageUnavailable = structuredClone(options.snapshot?.usageUnavailable ?? []);
this.#stateHistory = structuredClone(options.snapshot?.stateHistory ?? [{
revision: this.#port.snapshot().revision,
at: this.#createdAt,
@@ -1389,6 +1421,7 @@ export class CapabilityLiveSession {
currentAttemptId: this.#currentAttemptId,
terminalTurns: structuredClone(this.#terminalTurns),
usageLedger: structuredClone(this.#usageLedger),
...(this.#usageUnavailable.length === 0 ? {} : { usageUnavailable: structuredClone(this.#usageUnavailable) }),
stateHistory: structuredClone(this.#stateHistory),
workspaceDiffs: structuredClone(this.#workspaceDiffs),
workspaceFileReferences: structuredClone(this.#workspaceFileReferences),
@@ -1772,10 +1805,29 @@ export class CapabilityLiveSession {
...selected,
costNanodollars: Math.max(selected.costNanodollars, reported?.costNanodollars ?? 0),
};
if (
(selectedWithReportedCost?.inputTokens ?? 0) + (selectedWithReportedCost?.outputTokens ?? 0) === 0
&& !allowEmpty
) {
const missingTokens = (selectedWithReportedCost?.inputTokens ?? 0)
+ (selectedWithReportedCost?.outputTokens ?? 0) === 0;
const candidate = this.#config.acpxAgent;
if (missingTokens && this.#config.provider === "acpx"
&& (candidate === "pi" || candidate === "cursor" || candidate === "copilot")
&& this.#transportOptions.acpxCandidateProfile === candidate) {
// Native candidate wrappers can complete a turn without a usage receipt,
// including entitlement-denied turns. Retain the actual result for the
// oracle without inventing tokens, charges, or a successful model call.
if (!this.#usageUnavailable.some((entry) => entry.turnId === turnId)) {
this.#usageUnavailable.push({
turnId,
attemptId: this.#currentAttemptId,
agent: candidate,
reason: "provider_did_not_report_usage",
tokenUsage: null,
costNanodollars: null,
observedAt: this.#now().toISOString(),
});
}
return;
}
if (missingTokens && !allowEmpty) {
throw new Error(`capability_live_usage_missing:${JSON.stringify({
captured: captured !== undefined,
needsRead,