mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-22 16:51:28 +02:00
fix(runner): retain candidate results without invented usage receipts
Record explicit unavailable usage for opted-in ACP candidates and preserve their semantic oracle result without creating zero-token or zero-cost receipts. Qualified providers retain strict accounting; unknown candidate turns still count toward the turn limit. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
af06ada99f
commit
c41d134f72
4 files changed
+157
-12
No files matched your search
@@ -278,6 +278,33 @@ describe("eval-session usage", () => {
|
||||
})).toThrow("agent turn limit exceeded");
|
||||
});
|
||||
|
||||
it.each(["pi", "cursor", "copilot"] as const)("keeps %s oracle results when the current turn explicitly has unavailable usage", (agent) => {
|
||||
const parsed = parseEvalSessionRequest(request({
|
||||
provider: "acpx", acpxAgent: agent, model: "exact-provider-model",
|
||||
}), { candidateProfile: agent });
|
||||
const unavailable = {
|
||||
turnId: "turn-candidate", attemptId: "attempt-1", agent,
|
||||
reason: "provider_did_not_report_usage" as const,
|
||||
tokenUsage: null, costNanodollars: null, observedAt: "2026-09-28T19:18:10.000Z",
|
||||
};
|
||||
const snapshot = {
|
||||
config: { provider: "acpx", acpxAgent: agent },
|
||||
usageLedger: [], usageUnavailable: [unavailable],
|
||||
} as unknown as CapabilityLiveSessionSnapshot;
|
||||
const turn = { turnId: unavailable.turnId, status: "completed" as const, assistantText: "Upgrade your plan to continue", snapshot };
|
||||
expect(boundedEvalSessionUsage(parsed, turn)).toBeNull();
|
||||
expect(turn.assistantText).toBe("Upgrade your plan to continue");
|
||||
expect(snapshot.usageLedger).toEqual([]);
|
||||
expect(() => boundedEvalSessionUsage(parsed, { ...turn, turnId: "other-turn" }))
|
||||
.toThrow("omitted usage accounting");
|
||||
expect(() => boundedEvalSessionUsage(parsed, {
|
||||
...turn, snapshot: { ...snapshot, usageUnavailable: [{ ...unavailable, agent: agent === "pi" ? "cursor" : "pi" }] },
|
||||
})).toThrow("omitted usage accounting");
|
||||
expect(() => boundedEvalSessionUsage(parsed, {
|
||||
...turn, snapshot: { ...snapshot, usageUnavailable: [unavailable, { ...unavailable, turnId: "second-turn" }] },
|
||||
})).toThrow("agent turn limit exceeded");
|
||||
});
|
||||
|
||||
it("deduplicates receipts and applies the versioned model price", () => {
|
||||
const receipt = {
|
||||
receiptId: "receipt-1",
|
||||
|
||||
@@ -250,25 +250,37 @@ export function boundedEvalSessionUsage(
|
||||
if (turn.status !== "completed") {
|
||||
return usageIfAvailable(request, turn.snapshot);
|
||||
}
|
||||
const usage = evalSessionUsage(request.model, turn.snapshot);
|
||||
if (usage.agentTurns > request.limits.maxAgentTurns) {
|
||||
const unavailable = request.provider === "acpx"
|
||||
&& request.acpxAgent !== undefined
|
||||
&& ["pi", "cursor", "copilot"].includes(request.acpxAgent)
|
||||
&& turn.snapshot.config.provider === "acpx"
|
||||
&& turn.snapshot.config.acpxAgent === request.acpxAgent
|
||||
&& turn.snapshot.usageUnavailable?.some((entry) => entry.turnId === turn.turnId
|
||||
&& entry.agent === request.acpxAgent && entry.reason === "provider_did_not_report_usage");
|
||||
const usage = unavailable
|
||||
? usageIfAvailable(request, turn.snapshot)
|
||||
: evalSessionUsage(request.model, turn.snapshot);
|
||||
const unavailableTurns = unavailable
|
||||
? new Set(turn.snapshot.usageUnavailable?.map((entry) => entry.turnId)).size
|
||||
: 0;
|
||||
if ((usage?.agentTurns ?? 0) + unavailableTurns > request.limits.maxAgentTurns) {
|
||||
throw new Error("agent turn limit exceeded");
|
||||
}
|
||||
if (
|
||||
usage.estimatedCostNanodollars !== null &&
|
||||
usage !== null && usage.estimatedCostNanodollars !== null &&
|
||||
usage.estimatedCostNanodollars >
|
||||
request.limits.maxEstimatedCostNanodollars
|
||||
) {
|
||||
throw new Error("estimated cost limit exceeded");
|
||||
}
|
||||
if (
|
||||
usage.providerReportedCostNanodollars !== null &&
|
||||
usage !== null && usage.providerReportedCostNanodollars !== null &&
|
||||
usage.providerReportedCostNanodollars >
|
||||
request.limits.maxEstimatedCostNanodollars
|
||||
) {
|
||||
throw new Error("provider-reported cost limit exceeded");
|
||||
}
|
||||
return usage;
|
||||
return unavailable ? null : usage;
|
||||
}
|
||||
|
||||
async function closeSession(
|
||||
|
||||
@@ -75,6 +75,8 @@ interface FakeProviderState {
|
||||
holdAfterTool: boolean;
|
||||
closeError: Error | null;
|
||||
usageRunDelta: Record<string, unknown> | null;
|
||||
omitReadUsage?: boolean;
|
||||
assistantReply?: string;
|
||||
onTurnStart?: () => Promise<void>;
|
||||
onUsage?: (queue: AsyncNotifications, turnId: string) => void | Promise<void>;
|
||||
/** Delays the fake `turn/interrupt` reply, to model a slow transport round trip. */
|
||||
@@ -116,14 +118,14 @@ class FakeCapabilityCodexTransport implements CodexAppServerTransport {
|
||||
id: this.state.threadId,
|
||||
sessionId: this.state.providerSessionId,
|
||||
turns: [...this.state.turns].map(([id, status]) => ({ id, status })),
|
||||
tokenUsage: {
|
||||
...(this.state.omitReadUsage ? {} : { tokenUsage: {
|
||||
total: {
|
||||
inputTokens: this.state.nextTurn * 100,
|
||||
cachedInputTokens: this.state.nextTurn * 20,
|
||||
outputTokens: this.state.nextTurn * 10,
|
||||
reasoningOutputTokens: this.state.nextTurn * 2,
|
||||
},
|
||||
},
|
||||
} }),
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -260,7 +262,7 @@ class FakeCapabilityCodexTransport implements CodexAppServerTransport {
|
||||
params: {
|
||||
threadId: this.state.threadId,
|
||||
turnId,
|
||||
item: { id: `message-${turnId}`, type: "agentMessage", text: assistantText },
|
||||
item: { id: `message-${turnId}`, type: "agentMessage", text: this.state.assistantReply ?? assistantText },
|
||||
},
|
||||
});
|
||||
this.state.turns.set(turnId, "completed");
|
||||
@@ -1077,6 +1079,58 @@ describe("Capability live runnerd and Codex session", () => {
|
||||
await service.shutdown(session.id);
|
||||
});
|
||||
|
||||
it.each(["pi", "cursor", "copilot"] as const)("retains a completed %s result with explicit unavailable usage and no fabricated receipt", async (acpxAgent) => {
|
||||
const state = providerState();
|
||||
state.omitReadUsage = true;
|
||||
state.assistantReply = "Upgrade your plan to continue";
|
||||
state.onUsage = (queue, turnId) => queue.push({
|
||||
method: "turn/completed",
|
||||
params: { threadId: state.threadId, turn: { id: turnId, status: "completed" } },
|
||||
});
|
||||
const store = new InMemoryCapabilityLiveSessionStore();
|
||||
const service = new CapabilityLiveSessionService({
|
||||
store,
|
||||
transportFactory: fakeTransportFactory(state),
|
||||
transportOptions: { acpxCandidateProfile: acpxAgent },
|
||||
});
|
||||
const session = await service.create({
|
||||
provider: "acpx", acpxAgent,
|
||||
requestedModel: acpxAgent === "pi" ? "openrouter/deepseek/deepseek-v4-flash-0731" : "exact-model",
|
||||
});
|
||||
const result = await session.sendMessage("Orient to this task.");
|
||||
expect(result.status).toBe("completed");
|
||||
expect(result.assistantText).toBe(state.assistantReply);
|
||||
expect(result.snapshot.usageLedger).toEqual([]);
|
||||
expect(result.snapshot.usageUnavailable).toEqual([{
|
||||
turnId: result.turnId,
|
||||
attemptId: result.snapshot.currentAttemptId,
|
||||
agent: acpxAgent,
|
||||
reason: "provider_did_not_report_usage",
|
||||
tokenUsage: null,
|
||||
costNanodollars: null,
|
||||
observedAt: expect.any(String),
|
||||
}]);
|
||||
expect(result.snapshot.authorizationRecords.filter((entry) => entry.phase !== "exposure")).toEqual([]);
|
||||
await service.shutdown(session.id);
|
||||
expect((await store.load(session.id))?.usageUnavailable).toEqual(result.snapshot.usageUnavailable);
|
||||
expect((await store.load(session.id))?.process?.runnerExited).toBe(true);
|
||||
});
|
||||
|
||||
it("still rejects missing usage for a qualified provider", async () => {
|
||||
const state = providerState();
|
||||
state.omitReadUsage = true;
|
||||
state.onUsage = (queue, turnId) => queue.push({
|
||||
method: "turn/completed",
|
||||
params: { threadId: state.threadId, turn: { id: turnId, status: "completed" } },
|
||||
});
|
||||
const service = new CapabilityLiveSessionService({ transportFactory: fakeTransportFactory(state) });
|
||||
const session = await service.create();
|
||||
await expect(session.sendMessage("Orient to this task.")).rejects.toThrow("capability_live_usage_missing");
|
||||
expect(session.snapshot().usageLedger).toEqual([]);
|
||||
expect(session.snapshot().usageUnavailable).toBeUndefined();
|
||||
await service.shutdown(session.id);
|
||||
});
|
||||
|
||||
it("streams normalized usage before terminal with cumulative fallback and explicit run deltas", async () => {
|
||||
const state = providerState();
|
||||
const service = new CapabilityLiveSessionService({ transportFactory: fakeTransportFactory(state) });
|
||||
|
||||
@@ -199,6 +199,16 @@ export interface CapabilityLiveUsageReceipt {
|
||||
observedAt: string;
|
||||
}
|
||||
|
||||
export interface CapabilityLiveUnavailableUsage {
|
||||
turnId: string;
|
||||
attemptId: string;
|
||||
agent: "pi" | "cursor" | "copilot";
|
||||
reason: "provider_did_not_report_usage";
|
||||
tokenUsage: null;
|
||||
costNanodollars: null;
|
||||
observedAt: string;
|
||||
}
|
||||
|
||||
export interface CapabilityLiveStateRevision {
|
||||
revision: number;
|
||||
at: string;
|
||||
@@ -266,6 +276,8 @@ export interface CapabilityLiveSessionSnapshot {
|
||||
terminalTurns?: CapabilityLiveTurnTerminalFact[];
|
||||
/** Exact-once provider usage/cost receipts retained across every attempt. */
|
||||
usageLedger?: CapabilityLiveUsageReceipt[];
|
||||
/** Candidate turns without a provider receipt; these are never zero-valued receipts. */
|
||||
usageUnavailable?: CapabilityLiveUnavailableUsage[];
|
||||
/** Process-level evidence for an interrupted Codex semantic-call resume. */
|
||||
nativeResume?: {
|
||||
schema: "paperclip.runner.native-resume-proof/v1";
|
||||
@@ -768,6 +780,24 @@ export function assertCapabilityLiveSessionSnapshot(
|
||||
}
|
||||
usageReceiptIds.add(receiptId);
|
||||
}
|
||||
if (snapshot.usageUnavailable !== undefined && !Array.isArray(snapshot.usageUnavailable)) {
|
||||
throw new Error("capability_live_checkpoint_corrupt: invalid unavailable usage");
|
||||
}
|
||||
const unavailableTurnIds = new Set<string>();
|
||||
for (const value of snapshot.usageUnavailable ?? []) {
|
||||
const unavailable = record(value);
|
||||
const turnId = text(unavailable.turnId);
|
||||
if (config.provider !== "acpx" || unavailable.agent !== config.acpxAgent
|
||||
|| !["pi", "cursor", "copilot"].includes(text(unavailable.agent))
|
||||
|| !turnId || turnId.length > 512 || unavailableTurnIds.has(turnId)
|
||||
|| !text(unavailable.attemptId) || text(unavailable.attemptId).length > 512
|
||||
|| !text(unavailable.observedAt)
|
||||
|| unavailable.reason !== "provider_did_not_report_usage"
|
||||
|| unavailable.tokenUsage !== null || unavailable.costNanodollars !== null) {
|
||||
throw new Error("capability_live_checkpoint_corrupt: invalid unavailable usage");
|
||||
}
|
||||
unavailableTurnIds.add(turnId);
|
||||
}
|
||||
if (
|
||||
snapshot.stateHistory !== undefined &&
|
||||
(!Array.isArray(snapshot.stateHistory) || snapshot.stateHistory.length > 256)
|
||||
@@ -1147,6 +1177,7 @@ export class CapabilityLiveSession {
|
||||
readonly #attempts: CapabilityLiveAttemptSnapshot[];
|
||||
readonly #terminalTurns: CapabilityLiveTurnTerminalFact[];
|
||||
readonly #usageLedger: CapabilityLiveUsageReceipt[];
|
||||
readonly #usageUnavailable: CapabilityLiveUnavailableUsage[];
|
||||
readonly #stateHistory: CapabilityLiveStateRevision[];
|
||||
readonly #workspaceDiffs: CapabilityLiveWorkspaceDiffEntry[];
|
||||
readonly #workspaceFileReferences: CapabilityLiveWorkspaceFileReferenceEntry[];
|
||||
@@ -1204,6 +1235,7 @@ export class CapabilityLiveSession {
|
||||
this.#currentAttemptId = initialAttemptId;
|
||||
this.#terminalTurns = structuredClone(options.snapshot?.terminalTurns ?? []);
|
||||
this.#usageLedger = structuredClone(options.snapshot?.usageLedger ?? []);
|
||||
this.#usageUnavailable = structuredClone(options.snapshot?.usageUnavailable ?? []);
|
||||
this.#stateHistory = structuredClone(options.snapshot?.stateHistory ?? [{
|
||||
revision: this.#port.snapshot().revision,
|
||||
at: this.#createdAt,
|
||||
@@ -1389,6 +1421,7 @@ export class CapabilityLiveSession {
|
||||
currentAttemptId: this.#currentAttemptId,
|
||||
terminalTurns: structuredClone(this.#terminalTurns),
|
||||
usageLedger: structuredClone(this.#usageLedger),
|
||||
...(this.#usageUnavailable.length === 0 ? {} : { usageUnavailable: structuredClone(this.#usageUnavailable) }),
|
||||
stateHistory: structuredClone(this.#stateHistory),
|
||||
workspaceDiffs: structuredClone(this.#workspaceDiffs),
|
||||
workspaceFileReferences: structuredClone(this.#workspaceFileReferences),
|
||||
@@ -1772,10 +1805,29 @@ export class CapabilityLiveSession {
|
||||
...selected,
|
||||
costNanodollars: Math.max(selected.costNanodollars, reported?.costNanodollars ?? 0),
|
||||
};
|
||||
if (
|
||||
(selectedWithReportedCost?.inputTokens ?? 0) + (selectedWithReportedCost?.outputTokens ?? 0) === 0
|
||||
&& !allowEmpty
|
||||
) {
|
||||
const missingTokens = (selectedWithReportedCost?.inputTokens ?? 0)
|
||||
+ (selectedWithReportedCost?.outputTokens ?? 0) === 0;
|
||||
const candidate = this.#config.acpxAgent;
|
||||
if (missingTokens && this.#config.provider === "acpx"
|
||||
&& (candidate === "pi" || candidate === "cursor" || candidate === "copilot")
|
||||
&& this.#transportOptions.acpxCandidateProfile === candidate) {
|
||||
// Native candidate wrappers can complete a turn without a usage receipt,
|
||||
// including entitlement-denied turns. Retain the actual result for the
|
||||
// oracle without inventing tokens, charges, or a successful model call.
|
||||
if (!this.#usageUnavailable.some((entry) => entry.turnId === turnId)) {
|
||||
this.#usageUnavailable.push({
|
||||
turnId,
|
||||
attemptId: this.#currentAttemptId,
|
||||
agent: candidate,
|
||||
reason: "provider_did_not_report_usage",
|
||||
tokenUsage: null,
|
||||
costNanodollars: null,
|
||||
observedAt: this.#now().toISOString(),
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (missingTokens && !allowEmpty) {
|
||||
throw new Error(`capability_live_usage_missing:${JSON.stringify({
|
||||
captured: captured !== undefined,
|
||||
needsRead,
|
||||
|
||||
Reference in new issue
Block a user