fix(runner): fail eval budgets closed without losing outcomes

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/paperclip-runner/src/cli/eval-session-contract.test.ts

@@ -1,8 +1,10 @@
-import { chmod, mkdtemp, readFile, rm, stat } from "node:fs/promises";
+import { createHash } from "node:crypto";
+import { chmod, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
+import { createCapabilityFixtureState } from "../mock-core/capability-control-plane-types.js";
 
 import { parseNativeRuntimeContext } from "../contracts/runtime-context.js";
 import type { CapabilityLiveSessionSnapshot } from "../live/live-session.js";
@@ -12,6 +14,7 @@ import {
 } from "./eval-session-contract.js";
 import {
   boundedEvalSessionUsage,
+  runEvalSessionCli,
   evalRuntimeSystemInstructions,
   evalSessionProviderVersion,
   prepareEvalRuntimeContext,
@@ -292,7 +295,7 @@ describe("eval-session usage", () => {
       usageLedger: [], usageUnavailable: [unavailable],
     } as unknown as CapabilityLiveSessionSnapshot;
     const turn = { turnId: unavailable.turnId, status: "completed" as const, assistantText: "Upgrade your plan to continue", snapshot };
-    expect(boundedEvalSessionUsage(parsed, turn)).toBeNull();
+    expect(() => boundedEvalSessionUsage(parsed, turn)).toThrow("budget cost coverage is unavailable");
     expect(turn.assistantText).toBe("Upgrade your plan to continue");
     expect(snapshot.usageLedger).toEqual([]);
     expect(() => boundedEvalSessionUsage(parsed, { ...turn, turnId: "other-turn" }))
@@ -325,7 +328,7 @@ describe("eval-session usage", () => {
     } as unknown as CapabilityLiveSessionSnapshot;
     const turn = { turnId: "measured-turn", status: "completed" as const, assistantText: "Done", snapshot };
     expect(() => boundedEvalSessionUsage(parsed, turn)).toThrow("agent turn limit exceeded");
-    expect(boundedEvalSessionUsage({ ...parsed, limits: { ...parsed.limits, maxAgentTurns: 2 } }, turn)).toBeNull();
+    expect(() => boundedEvalSessionUsage({ ...parsed, limits: { ...parsed.limits, maxAgentTurns: 2 } }, turn)).toThrow("budget cost coverage is unavailable");
     expect(boundedEvalSessionUsage(parsed, {
       ...turn,
       snapshot: { ...snapshot, usageUnavailable: [{ ...unavailable, turnId: "measured-turn" }] },
@@ -385,6 +388,10 @@ describe("eval-session usage", () => {
       estimatedCostNanodollars: null, pricingVersion: null, ratesUsdPerMillionTokens: null,
       costCoverage: "unpriced",
     });
+    const parsed = parseEvalSessionRequest(request({ provider: "acpx", acpxAgent: agent, model: "exact-provider-model[context=272k]" }), { candidateProfile: agent as "pi" | "cursor" | "copilot" });
+    expect(() => boundedEvalSessionUsage(parsed, {
+      turnId: "candidate-turn", status: "completed", assistantText: "Completed provider response", snapshot,
+    })).toThrow("budget cost coverage is unavailable");
     const priced = evalSessionUsage("openrouter/deepseek/deepseek-v4-flash-0731", snapshot);
     expect(priced.providerReportedCostNanodollars).toBeNull();
     expect(priced.costCoverage).toBe("estimated");
@@ -396,3 +403,60 @@ describe("eval-session usage", () => {
     } as unknown as CapabilityLiveSessionSnapshot)).toThrow("model pricing unavailable");
   });
 });
+
+describe("eval-session budget settlement", () => {
+  it.each(["no_receipt", "unpriced", "mixed", "over_limit"] as const)("retains the completed provider outcome while failing %s accounting", async (kind) => {
+    const workspace = await mkdtemp(join(tmpdir(), "eval-budget-settlement-"));
+    try {
+      const binary = join(workspace, "runnerd");
+      await writeFile(binary, "unused fake runner");
+      const model = kind === "unpriced" || kind === "no_receipt" ? "exact-unpriced-candidate" : "gpt-5.6-sol";
+      const unavailable = { turnId: kind === "mixed" ? "prior-turn" : "turn-1", attemptId: "attempt-1", agent: "copilot", reason: "provider_did_not_report_usage", tokenUsage: null, costNanodollars: null, observedAt: "2026-09-28T19:00:00.000Z" };
+      const state = JSON.stringify(createCapabilityFixtureState({}));
+      const snapshot = {
+        sessionId: "session-1", revision: 1, providerThreadId: "provider-1", providerSessionId: "provider-1",
+        providerModel: { id: model, provider: "github" }, status: "idle", activeTurnId: null,
+        createdAt: "2026-09-28T19:00:00.000Z", updatedAt: "2026-09-28T19:00:01.000Z",
+        authority: { companyId: "company-1", actorId: "actor-1", taskId: "task-1", runId: "run-1", sessionId: "session-1", scenarioId: "budget-test" },
+        config: { provider: "acpx", acpxAgent: "copilot", driver: "acpx_runtime", seedState: state, workingDirectory: workspace, scenario: { id: "budget-test" }, capabilities: [], explicitClaims: [], turnTimeoutMs: 1000 },
+        mockState: state, process: null, networkEvidence: { realPaperclipRequests: 0, childPaperclipEnvironmentKeys: [] },
+        transcript: [{ id: "assistant-1", role: "assistant", text: "Retained actual provider response", turnId: "turn-1", at: "2026-09-28T19:00:01.000Z" }],
+        evidence: [], authorizationRecords: [], attempts: [], terminalTurns: [{ turnId: "turn-1", status: "completed" }],
+        usageLedger: kind === "no_receipt" ? [] : [{ receiptId: "receipt-1", turnId: "turn-1", providerCalls: 1, providerRequests: 1, inputTokens: 100, outputTokens: kind === "over_limit" ? 1_000_000 : 10, cachedInputTokens: 0, reasoningTokens: 0, costNanodollars: 0 }],
+        usageUnavailable: kind === "no_receipt" || kind === "mixed" ? [unavailable] : [],
+      } as unknown as CapabilityLiveSessionSnapshot;
+      const sendMessage = vi.fn(async () => ({ turnId: "turn-1", status: "completed", assistantText: "Retained actual provider response", snapshot }));
+      const completeAttempt = vi.fn(async () => undefined);
+      const shutdown = vi.fn(async () => { snapshot.status = "closed"; });
+      const input = request({ provider: "acpx", acpxAgent: "copilot", model,
+        runnerd: { path: binary, sha256: createHash("sha256").update("unused fake runner").digest("hex") },
+        session: { workingDirectory: workspace }, limits: { turnTimeoutMs: 1000, maxAgentTurns: 2, maxEstimatedCostNanodollars: 100_000_000 },
+      });
+      const requestPath = join(workspace, "request.json");
+      const outputPath = join(workspace, "output.json");
+      await writeFile(requestPath, JSON.stringify(input));
+      const exitCode = await runEvalSessionCli(["--request", requestPath, "--output", outputPath, "--candidate-profile", "copilot"], {
+        serviceFactory: () => ({ create: async () => ({ sendMessage, completeAttempt, shutdown, snapshot: () => snapshot }) }) as never,
+      });
+      const artifact = JSON.parse(await readFile(outputPath, "utf8"));
+      expect(exitCode).toBe(2);
+      expect(artifact).not.toHaveProperty("infrastructureError");
+      expect(artifact.accountingFailure).toMatchObject({ class: kind === "over_limit" ? "provider_budget_reached" : "provider_budget_coverage_unknown", retryable: false });
+      expect(artifact.turn).toMatchObject({ status: "completed", assistantText: "Retained actual provider response" });
+      expect(artifact.snapshot.status).toBe("closed");
+      expect(artifact.acpxProfile.agent).toBe("copilot");
+      expect(artifact.devtools).toBeDefined();
+      expect(artifact.issueThread).toBeDefined();
+      if (kind === "no_receipt") expect(artifact).not.toHaveProperty("usage");
+      else expect(artifact.usage.providerReportedCostNanodollars).toBeNull();
+      expect(sendMessage).toHaveBeenCalledTimes(1);
+      expect(shutdown).toHaveBeenCalledTimes(1);
+      expect(completeAttempt).toHaveBeenCalledWith("failed", artifact.accountingFailure.class);
+    } finally {
+      for (const entry of await readdir(workspace)) {
+        if (entry.startsWith(".paperclip-eval-runtime-context-")) await chmod(join(workspace, entry, "instructions"), 0o700).catch(() => undefined);
+      }
+      await rm(workspace, { recursive: true, force: true });
+    }
+  });
+});

packages/paperclip-runner/src/cli/eval-session.ts

@@ -192,12 +192,22 @@ export function evalSessionProviderVersion(
   return null;
 }
 
+class EvalSessionBudgetError extends Error {
+  constructor(message: string, readonly coverageUnknown = false) {
+    super(message);
+    this.name = "EvalSessionBudgetError";
+  }
+}
+
 function failureClass(error: unknown): {
   class: string;
   category: string;
   retryable: boolean;
   diagnostics: Record<string, never>;
 } {
+  if (error instanceof EvalSessionBudgetError && error.coverageUnknown) {
+    return { class: "provider_budget_coverage_unknown", category: "provider_budget", retryable: false, diagnostics: {} };
+  }
   const message = error instanceof Error ? error.message : String(error);
   if (/timed? ?out|timeout/i.test(message)) {
     return {
@@ -268,23 +278,26 @@ export function boundedEvalSessionUsage(
     : evalSessionUsage(request.model, turn.snapshot);
   const unavailableTurns = unavailableTurnIds.size;
   if ((usage?.agentTurns ?? 0) + unavailableTurns > request.limits.maxAgentTurns) {
-    throw new Error("agent turn limit exceeded");
+    throw new EvalSessionBudgetError("agent turn limit exceeded");
   }
   if (
     usage !== null && usage.estimatedCostNanodollars !== null &&
     usage.estimatedCostNanodollars >
     request.limits.maxEstimatedCostNanodollars
   ) {
-    throw new Error("estimated cost limit exceeded");
+    throw new EvalSessionBudgetError("estimated cost limit exceeded");
   }
   if (
     usage !== null && usage.providerReportedCostNanodollars !== null &&
     usage.providerReportedCostNanodollars >
     request.limits.maxEstimatedCostNanodollars
   ) {
-    throw new Error("provider-reported cost limit exceeded");
+    throw new EvalSessionBudgetError("provider-reported cost limit exceeded");
   }
-  return unavailable ? null : usage;
+  if (unavailable || usage === null || (usage.estimatedCostNanodollars === null && usage.providerReportedCostNanodollars === null)) {
+    throw new EvalSessionBudgetError("budget cost coverage is unavailable for one or more completed turns", true);
+  }
+  return usage;
 }
 
 async function closeSession(
@@ -369,10 +382,21 @@ export async function runEvalSessionCli(
     } as unknown as CreateCapabilityLiveSessionInput;
     session = await service.create(createInput);
     turn = await session.sendMessage(request.prompt);
-    const usage = boundedEvalSessionUsage(request, turn);
+    let usage: EvalSessionUsage | null;
+    let accountingError: EvalSessionBudgetError | null = null;
+    try {
+      usage = boundedEvalSessionUsage(request, turn);
+    } catch (error) {
+      if (!(error instanceof EvalSessionBudgetError)) throw error;
+      // A completed provider outcome remains inspectable even when its budget
+      // cannot be verified. Never turn unknown cost into a successful CLI run.
+      accountingError = error;
+      usage = usageIfAvailable(request, turn.snapshot);
+    }
     await session.completeAttempt(
-      turn.status === "completed" ? "succeeded" : "failed",
-      turn.status === "completed" ? null : `provider_turn_${turn.status}`,
+      accountingError === null && turn.status === "completed" ? "succeeded" : "failed",
+      accountingError !== null ? failureClass(accountingError).class
+        : turn.status === "completed" ? null : `provider_turn_${turn.status}`,
     );
     await closeSession(session, "eval session complete");
     snapshot = session.snapshot();
@@ -411,6 +435,10 @@ export async function runEvalSessionCli(
         : {}),
       turn,
       snapshot,
+      ...(accountingError === null ? {} : {
+        accountingError: accountingError.message,
+        accountingFailure: failureClass(accountingError),
+      }),
       devtools: projectCapabilityDevtools(snapshot),
       issueThread: projectCapabilityIssueThread({
         snapshot,
@@ -424,7 +452,7 @@ export async function runEvalSessionCli(
         durationMs: Date.now() - startedAtMs,
       },
     }, null, 2)}\n`);
-    return 0;
+    return accountingError === null ? 0 : 2;
   } catch (error) {
     if (session !== null) {
       snapshot = session.snapshot();