fix: preserve native recovery and settle the latest durable receipt

Keep fresh budget admission blocked without cancelling the native recovery needed to close accounting. Drain every matching disk receipt under the accounting lock and retain files until ledger commit. Add PostgreSQL rollback, backlog, movement, and budget-scope regressions.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/COST-ACCOUNTING.md

@@ -73,6 +73,8 @@ The spool lives at `<instance-root>/accounting-receipts`. Directories use mode 0
 
 Before replacing a stopped run's recorder, recovery saves every pending spool receipt for that company and run, independently of the bounded startup sweep. A save failure blocks replacement and retains the old source for retry. Native recovery then restores its latest journal snapshot; subsequent cumulative run usage replaces that snapshot rather than adding it twice.
 
+Settlement also drains every pending spool receipt for the run under the company accounting lock before checking completeness or acknowledging usage. A newer partial receipt prevents settlement of an older complete snapshot. Recovered journal writes and ledger settlement share a transaction; spool files are removed only after its commit, so a failed charge or projection write leaves the original evidence available for retry. A concurrent replay rename causes a rescan; repeated file movement defers settlement.
+
 The spool survives process death on persistent local storage, including the tested boundary before the first database write. It does not survive destruction of that storage, and cannot recover provider activity that was never emitted as usage. Back up persistent instance storage together with PostgreSQL. No arbitrary provider-log scraping or automatic provider re-execution occurs during accounting recovery.
 
 Board-only routes under `/api/companies/:companyId/accounting` provide:
@@ -181,7 +183,10 @@ Failed native turns persist observed run-delta usage before semantic-result
 finalization. A matching terminal failure, cancellation, or interruption closes
 that usage snapshot; ledger acknowledgement still waits until the native coordinator
 has a result or a terminal failure. A retryable failed heartbeat keeps its
-reservation and can resume. Replacing its recorder restores the native run's
+reservation and can resume. An open native coordinator's pending accounting
+blocks fresh admission without pausing or cancelling its own recovery. Actual
+budget overruns, unpriced charges, and closed runs missing accounting still
+enforce hard stops; manual pauses remain unchanged. Replacing its recorder restores the native run's
 cumulative snapshot; recovery does not add the same tokens twice, and an empty
 final result cannot erase earlier observed usage. Missing terminal evidence and
 session-only totals remain pending.

server/src/__tests__/cost-accounting-operations-edges.test.ts

@@ -7,7 +7,7 @@ import { and, eq, sql } from "drizzle-orm";
 import { accountingRuntimeBaselines, agents, agentRuntimeState, billingInvoiceLines, budgetPolicies, budgetReservations, companies, costEvents, createDb, heartbeatRuns, nativeRunFinalizations, issues, projects, runUsageReceipts } from "@paperclipai/db";
 import { accountingIntegrityService } from "../services/accounting-integrity.js";
 import { billingReconciliationService } from "../services/billing-reconciliation.js";
-import { budgetService } from "../services/budgets.js";
+import { budgetService, type BudgetEnforcementScope } from "../services/budgets.js";
 import { reserveRunBudget } from "../services/budget-reservations.js";
 import { costService } from "../services/costs.js";
 import { financeService } from "../services/finance.js";
@@ -150,6 +150,136 @@ const support = await getEmbeddedPostgresTestSupport();
     expect((await costService(db).summary(f.company.id)).spendCents).toBe(22);
   });
 
+  it.each(["company", "agent", "project"] as const)("blocks fresh %s work without pausing native recovery", async (scopeType) => {
+    const f = await fixture(), cancelWorkForScope = vi.fn(async (_scope: BudgetEnforcementScope) => {});
+    const budgets = budgetService(db, { cancelWorkForScope });
+    const scopeId = f[scopeType].id;
+    await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: 100, reservationCents: "20" }, "board");
+    const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Recover under budget" }).returning();
+    await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, f.run.id));
+    const original = await reserveRunBudget(db, f.company.id, f.run.id, f.project.id);
+    await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: f.run.id, issueId: issue.id, phase: "retryable_failure" });
+    const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: f.run.id, adapterType: "paperclip_runner" }, path.join(directory, randomUUID()));
+    await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 100, outputTokens: 10 }, costUsd: 0.1 });
+    await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
+    // Also recover a budget-owned pause left by the previous implementation.
+    const table = { company: companies, agent: agents, project: projects }[scopeType];
+    await db.update(table).set({ pauseReason: "budget", ...(scopeType === "project" ? {} : { status: "paused" }) }).where(eq(table.id, scopeId));
+    await reconcileRunCosts(db, { cancelWorkForScope });
+    expect((await db.select().from(table).where(eq(table.id, scopeId)))[0].pauseReason).toBeNull();
+    expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("idle");
+    expect(cancelWorkForScope.mock.calls.filter(([scope]) => scope.companyId === f.company.id)).toEqual([]);
+    expect((await budgets.getInvocationBlock(f.company.id, f.agent.id, { projectId: f.project.id }))?.reason).toContain("native run recovers");
+    const [fresh] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "running" }).returning();
+    await expect(reserveRunBudget(db, f.company.id, fresh.id, f.project.id)).rejects.toThrow("native run recovers");
+    expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountedAt).toBeNull();
+    await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, f.run.id));
+    await db.update(nativeRunFinalizations).set({ leaseOwner: "recovery", leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.run.id));
+    expect(await reserveRunBudget(db, f.company.id, f.run.id, f.project.id, {}, "recovery"))
+      .toMatchObject({ id: original.id, reused: true, amountCents: "20.0000000", state: "held" });
+    expect(await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id))).toHaveLength(1);
+    await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 150, outputTokens: 15 }, costUsd: 0.15 });
+    await db.update(nativeRunFinalizations).set({ phase: "terminal_failure" }).where(eq(nativeRunFinalizations.runId, f.run.id));
+    await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, f.run.id));
+    expect(await accountRunCost(db, f.run.id)).toBe(true);
+    expect(await accountRunCost(db, f.run.id)).toBe(false);
+    expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id)))[0]).toMatchObject({ inputTokens: 150, costCents: 15 });
+    expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("settled");
+    expect(await budgets.getInvocationBlock(f.company.id, f.agent.id, { projectId: f.project.id })).toBeNull();
+  });
+
+  it.each(["closed", "over_budget", "unpriced", "manual"])("preserves %s stops while native accounting is unfinished", async (stop) => {
+    const f = await fixture(), budgets = budgetService(db);
+    await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100 }, "board");
+    const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Pending native run" }).returning();
+    await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, f.run.id));
+    await reserveRunBudget(db, f.company.id, f.run.id, null);
+    await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: f.run.id, issueId: issue.id, phase: stop === "closed" ? "terminal_failure" : "retryable_failure" });
+    await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
+    if (stop === "over_budget") await event(f, { costCents: 100 });
+    if (stop === "unpriced") await event(f, { costCents: 0, costStatus: "unpriced" });
+    if (stop === "manual") await db.update(agents).set({ status: "paused", pauseReason: "manual" }).where(eq(agents.id, f.agent.id));
+    await budgets.reconcilePolicies();
+    expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0])
+      .toMatchObject({ status: "paused", pauseReason: stop === "manual" ? "manual" : "budget" });
+    expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("held");
+    expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountedAt).toBeNull();
+  });
+
+  it.each(["complete", "partial", "receipt_failure", "ledger_failure", "cleanup_failure", "moved", "unstable_spool"])("settles the newest run receipt beyond a full replay batch (%s)", async (scenario) => {
+    const f = await fixture(), other = await fixture(), spool = usageReceiptSpoolPath();
+    const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: f.run.id, adapterType: "process" });
+    await reserveRunBudget(db, f.company.id, f.run.id, null);
+    await recorder.capture({ complete: true, usage: { inputTokens: 100, outputTokens: 10 }, costUsd: 0.1 });
+    const unavailable = vi.spyOn(db, "transaction").mockRejectedValue(new Error("Database unavailable"));
+    try { await recorder.capture({ complete: scenario !== "partial", usage: { inputTokens: 200, outputTokens: 20 }, costUsd: 0.2 }); }
+    finally { unavailable.mockRestore(); }
+    const [pending] = (await fs.readdir(spool)).filter(name => name.endsWith(".json"));
+    const file = path.join(spool, `zzz-${pending}`);
+    await fs.rename(path.join(spool, pending), file);
+    const unrelated: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", id: randomUUID(), companyId: other.company.id,
+      runId: other.run.id, sourceId: randomUUID(), sequence: 1, receivedAt: new Date().toISOString(), adapterType: "process", receipt: { complete: false } };
+    for (let index = 0; index < 101; index++) await fs.writeFile(path.join(spool, `000-${index}.json`), JSON.stringify({ ...unrelated, id: randomUUID(), sequence: index + 1 }));
+    expect(await replayUsageReceipts(db)).toEqual({ replayed: 100, failed: 0 });
+    expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.inputTokens).toBe(100);
+    await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
+    if (scenario === "unstable_spool") {
+      const stat = fs.lstat.bind(fs);
+      const unstable = vi.spyOn(fs, "lstat").mockImplementation(async (target, ...args) => {
+        if (target === file) throw Object.assign(new Error("Concurrent rename"), { code: "ENOENT" });
+        return stat(target, ...args);
+      });
+      try { await expect(accountRunCost(db, f.run.id)).rejects.toThrow("retry required"); }
+      finally { unstable.mockRestore(); }
+      expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountedAt).toBeNull();
+      expect(JSON.parse(await fs.readFile(file, "utf8")).receipt.usage.inputTokens).toBe(200);
+    }
+    if (scenario.endsWith("_failure") && scenario !== "cleanup_failure") {
+      const table = scenario === "receipt_failure" ? "run_usage_receipts" : "cost_events";
+      await db.execute(sql`create function reject_settlement() returns trigger language plpgsql as $$ begin raise exception 'settlement save failed'; end $$`);
+      await db.execute(sql.raw(`create trigger reject_settlement before insert on ${table} for each row execute function reject_settlement()`));
+      try {
+        await expect(accountRunCost(db, f.run.id)).rejects.toThrow();
+        expect(JSON.parse(await fs.readFile(file, "utf8")).receipt.usage.inputTokens).toBe(200);
+        expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0])
+          .toMatchObject({ costAccountedAt: null, usageJson: { inputTokens: 100 } });
+        expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toEqual([]);
+        expect(await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, f.run.id))).toHaveLength(1);
+        expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("held");
+      } finally {
+        await db.execute(sql.raw(`drop trigger reject_settlement on ${table}`));
+        await db.execute(sql`drop function reject_settlement()`);
+      }
+    }
+    const remove = fs.rm.bind(fs), stat = fs.lstat.bind(fs);
+    let moved = false;
+    const rm = scenario === "cleanup_failure" ? vi.spyOn(fs, "rm").mockImplementation(async (target, options) => {
+      if (target === file) throw new Error("Temporary cleanup failure");
+      return remove(target, options);
+    }) : null;
+    const lstat = scenario === "moved" ? vi.spyOn(fs, "lstat").mockImplementation(async (target, ...args) => {
+      if (target === file && !moved) { moved = true; await fs.rename(file, `${file.slice(0, -5)}-moved.json`); }
+      return stat(target, ...args);
+    }) : null;
+    try { await reconcileRunCosts(db); } finally { rm?.mockRestore(); lstat?.mockRestore(); }
+    if (scenario === "partial") {
+      expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0])
+        .toMatchObject({ costAccountedAt: null, usageJson: { inputTokens: 200, accountingReceiptReady: false } });
+      expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toEqual([]);
+      await recorder.capture({ complete: true, usage: { inputTokens: 300, outputTokens: 30 }, costUsd: 0.3 });
+      expect(await accountRunCost(db, f.run.id)).toBe(true);
+    }
+    expect(await accountRunCost(db, f.run.id)).toBe(false);
+    const expected = scenario === "partial" ? 300 : 200;
+    const charges = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id));
+    expect(charges).toHaveLength(1);
+    expect(charges[0]).toMatchObject({ inputTokens: expected, outputTokens: expected / 10, costCents: expected / 10 });
+    expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("settled");
+    expect(await replayUsageReceipts(db)).toEqual({ replayed: scenario === "cleanup_failure" ? 2 : 1, failed: 0 });
+    expect((await costService(db).summary(f.company.id)).spendCents).toBe(expected / 10);
+    expect(await fs.readdir(spool)).toEqual([]);
+  });
+
   it("leaves unrelated corrupt spool evidence alone and aborts handoff on unreadable or invalid matching receipts", async () => {
     const f = await fixture(), spool = path.join(directory, randomUUID());
     const input = { companyId: f.company.id, runId: f.run.id, adapterType: "paperclip_runner" };

server/src/services/budgets.ts

@@ -9,6 +9,7 @@ import {
   companies,
   costEvents,
   heartbeatRuns,
+  nativeRunFinalizations,
   projects,
 } from "@paperclipai/db";
 import type {
@@ -152,7 +153,7 @@ export async function computeObservedSpend(
   db: Db,
   policy: Pick<PolicyRow, "companyId" | "scopeType" | "scopeId" | "windowKind" | "metric">,
 ) {
-  if (policy.metric !== "billed_cents") return { total: 0, totalExact: "0.0000000", unpricedEventCount: 0, pendingRunCount: 0 };
+  if (policy.metric !== "billed_cents") return { total: 0, totalExact: "0.0000000", unpricedEventCount: 0, pendingRunCount: 0, recoveringRunCount: 0 };
 
   const conditions = [eq(costEvents.companyId, policy.companyId)];
   if (policy.scopeType === "agent") conditions.push(eq(costEvents.agentId, policy.scopeId));
@@ -178,8 +179,17 @@ export async function computeObservedSpend(
   if (policy.windowKind === "calendar_month_utc") {
     pendingConditions.push(sql`coalesce(${heartbeatRuns.finishedAt}, ${heartbeatRuns.createdAt}) >= ${start.toISOString()}::timestamptz`, sql`coalesce(${heartbeatRuns.finishedAt}, ${heartbeatRuns.createdAt}) < ${end.toISOString()}::timestamptz`);
   }
-  const [pending] = await db.select({ count: sql<number>`count(*)::int` }).from(heartbeatRuns).where(and(...pendingConditions));
-  return { total: Number(row?.total ?? 0), totalExact: normalizeCents(row?.total ?? 0), unpricedEventCount: Number(row?.unpricedEventCount ?? 0), pendingRunCount: pending?.count ?? 0 };
+  const [pending] = await db.select({
+    count: sql<number>`count(*)::int`,
+    recovering: sql<number>`count(*) filter (where ${heartbeatRuns.runtimeMode} = 'native'
+      and exists (select 1 from ${nativeRunFinalizations}
+        where ${nativeRunFinalizations.runId} = ${heartbeatRuns.id}
+          and ${nativeRunFinalizations.companyId} = ${heartbeatRuns.companyId}
+          and ${nativeRunFinalizations.resultId} is null
+          and ${nativeRunFinalizations.phase} <> 'terminal_failure'))::int`,
+  }).from(heartbeatRuns).where(and(...pendingConditions));
+  return { total: Number(row?.total ?? 0), totalExact: normalizeCents(row?.total ?? 0), unpricedEventCount: Number(row?.unpricedEventCount ?? 0),
+    pendingRunCount: pending?.count ?? 0, recoveringRunCount: pending?.recovering ?? 0 };
 }
 
 async function computeObservedAmount(db: Db, policy: PolicyRow) {
@@ -190,7 +200,7 @@ async function reached(db: Db, policy: PolicyRow, threshold: number) {
 }
 function observedBlocks(policy: PolicyRow, observed: Awaited<ReturnType<typeof computeObservedSpend>>) {
   if (!policy.isActive || !policy.hardStopEnabled || policy.amount <= 0) return false;
-  return compareCents(observed.totalExact, policy.amount) >= 0 || observed.pendingRunCount > 0 || (policy.unpricedUsagePolicy !== "allow" && observed.unpricedEventCount > 0);
+  return compareCents(observed.totalExact, policy.amount) >= 0 || (observed.pendingRunCount - observed.recoveringRunCount) > 0 || (policy.unpricedUsagePolicy !== "allow" && observed.unpricedEventCount > 0);
 }
 async function policyBlocks(db: Db, policy: PolicyRow) {
   if (!policy.isActive || !policy.hardStopEnabled || policy.amount <= 0) return false;
@@ -552,10 +562,11 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
     const policies = await db.select().from(budgetPolicies).where(and(
       eq(budgetPolicies.companyId, companyId), eq(budgetPolicies.scopeType, scopeType), eq(budgetPolicies.scopeId, scopeId),
     ));
+    let recoveringPolicy: PolicyRow | undefined;
     for (const policy of policies) {
-      const observed = await computeObservedAmount(db, policy);
-      if (await policyBlocks(db, policy)) {
-        await createIncidentIfNeeded(policy, "hard", observed);
+      const observed = await computeObservedSpend(db, policy);
+      if (observedBlocks(policy, observed)) {
+        await createIncidentIfNeeded(policy, "hard", observed.total);
         await pauseAndCancelScopeForBudget(policy);
         const title = scopeType === "company" ? "Company" : scopeType === "agent" ? "Agent" : "Project";
         if ((await computeObservedSpend(db, policy)).pendingRunCount > 0) {
@@ -568,6 +579,9 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
           ? "Project cannot start work because its budget hard-stop is still exceeded."
           : `${title} is paused because its budget hard-stop was reached.` };
       }
+      // An open native coordinator must finish the original run's accounting.
+      // Deny fresh work without pausing/cancelling that recovery or its hold.
+      if (policy.isActive && policy.hardStopEnabled && policy.amount > 0 && observed.recoveringRunCount > 0) recoveringPolicy ??= policy;
       // A closed UTC window, inactive policy, or budget raise releases only
       // budget-owned pauses. Manual pauses remain operator decisions.
       await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "hard");
@@ -584,6 +598,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
       return { scopeType, scopeId, scopeName: scope.name, reason: "Budget pause requires a policy or an explicit operator resume." };
     }
     if (scope.pauseReason === "budget" && policies[0]) await resumeScopeFromBudget(policies[0]);
+    if (recoveringPolicy) return { scopeType, scopeId, scopeName: scope.name,
+      reason: "New work must wait while a native run recovers its unfinished accounting." };
     return null;
   }
 

server/src/services/run-cost-accounting.ts

@@ -5,6 +5,8 @@ import { withAccountingTransaction } from "./accounting-transaction.js";
 import { createCostEventInTransaction } from "./costs.js";
 import { budgetService, deliverBudgetEnforcement, type BudgetServiceHooks } from "./budgets.js";
 import { logger } from "../middleware/logger.js";
+import { promises as fs } from "node:fs";
+import { recoverPendingRunUsageReceipts } from "./usage-receipts.js";
 
 const terminalStatuses = ["succeeded", "failed", "timed_out", "cancelled", "interrupted"];
 const object = (value: unknown): Record<string, unknown> => value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
@@ -16,13 +18,19 @@ const amount = (value: unknown) => typeof value === "number" && Number.isFinite(
 export async function accountRunCost(db: Db, runId: string, hooks: BudgetServiceHooks = {}) {
   const [identity] = await db.select({ companyId: heartbeatRuns.companyId }).from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
   if (!identity) return false;
+  let recoveredFiles: string[] = [];
   const accounted = await withAccountingTransaction(db, identity.companyId, async (tx, publications) => {
-    const [run] = await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).for("update");
+    let [run] = await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).for("update");
     if (!run?.costAccountingPending || !terminalStatuses.includes(run.status)) return false;
     if (run.costAccountedAt) {
       await tx.update(heartbeatRuns).set({ costAccountingPending: false }).where(eq(heartbeatRuns.id, run.id));
       return false;
     }
+    // The bounded global replay can leave a newer receipt behind an older
+    // complete snapshot. Drain this run under the accounting lock before
+    // deciding completeness, and retain disk evidence until the outer commit.
+    recoveredFiles = await recoverPendingRunUsageReceipts(tx, { ...identity, runId }, undefined, true);
+    [run] = await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
     const preProviderFailure = object(object(run.resultJson).executionRecovery).providerWorkStarted === false
       || object(run.usageJson).accountingProviderWorkStarted === false;
     // A failed native turn is not necessarily the end of this heartbeat: the
@@ -118,6 +126,10 @@ export async function accountRunCost(db: Db, runId: string, hooks: BudgetService
     await tx.update(heartbeatRuns).set({ costAccountingPending: false, costAccountedAt: new Date(), accountingProjectionVersion: "v2", accountingLastError: null, accountingLastAttemptAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
     return true;
   });
+  for (const file of recoveredFiles) {
+    try { await fs.rm(file, { force: true }); }
+    catch (error) { logger.warn({ err: error, runId }, "Accounting committed; recovered receipt cleanup will retry"); }
+  }
   await deliverBudgetEnforcement(db, hooks, identity.companyId);
   return accounted;
 }

server/src/services/usage-receipts.ts

@@ -113,28 +113,44 @@ export async function replayUsageReceipts(db: Db, directory = usageReceiptSpoolP
   return { replayed, failed };
 }
 
-async function recoverPendingRunUsageReceipts(db: Db, input: { companyId: string; runId: string }, directory: string) {
-  // Startup's bounded sweep may not reach this run. Its stopped controller's
-  // durable evidence must reach the journal before we fence that source out.
-  // A failed save deliberately aborts replacement so recovery can retry.
-  for (const name of (await fs.readdir(directory)).filter(name => name.endsWith(".json"))) {
-    const file = path.join(directory, name);
-    let raw: unknown;
-    try {
-      const info = await fs.lstat(file);
-      if (!info.isFile() || info.size > 1024 * 1024) continue;
-      raw = JSON.parse(await fs.readFile(file, "utf8"));
-    } catch (error) {
-      // The background replayer can move/remove a file while we scan. Invalid
-      // JSON remains on disk for its normal recovery/error reporting path.
-      if ((error as NodeJS.ErrnoException).code === "ENOENT" || error instanceof SyntaxError) continue;
-      throw error;
+export async function recoverPendingRunUsageReceipts(db: Db, input: { companyId: string; runId: string }, directory = usageReceiptSpoolPath(), retainFiles = false) {
+  // The bounded sweep may not reach this run. Its durable evidence must
+  // reach the journal before source replacement or ledger acknowledgement.
+  // A failed save aborts the operation so recovery can retry.
+  const recovered = new Set<string>();
+  // A concurrent failed replay can rename a file after releasing its company
+  // lock. Rescan disappearing paths rather than certify an incomplete scan.
+  for (let scan = 0; scan < 3; scan++) {
+    let names: string[];
+    try { names = await fs.readdir(directory); }
+    catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return [...recovered]; throw error; }
+    let moved = false;
+    for (const name of names.filter(name => name.endsWith(".json"))) {
+      const file = path.join(directory, name);
+      if (recovered.has(file)) continue;
+      let raw: unknown;
+      try {
+        const info = await fs.lstat(file);
+        if (!info.isFile() || info.size > 1024 * 1024) continue;
+        raw = JSON.parse(await fs.readFile(file, "utf8"));
+      } catch (error) {
+        // The background replayer can move/remove a file while we scan. Invalid
+        // JSON remains on disk for its normal recovery/error reporting path.
+        if ((error as NodeJS.ErrnoException).code === "ENOENT") { moved = true; continue; }
+        if (error instanceof SyntaxError) continue;
+        throw error;
+      }
+      if (!raw || typeof raw !== "object" || !("companyId" in raw) || !("runId" in raw)
+        || raw.companyId !== input.companyId || raw.runId !== input.runId) continue;
+      await persistUsageReceipt(db, envelopeSchema.parse(raw));
+      // Settlement uses the caller's transaction: deleting before its commit
+      // would lose the only durable evidence if the later ledger write fails.
+      recovered.add(file);
+      if (!retainFiles) await fs.rm(file, { force: true });
     }
-    if (!raw || typeof raw !== "object" || !("companyId" in raw) || !("runId" in raw)
-      || raw.companyId !== input.companyId || raw.runId !== input.runId) continue;
-    await persistUsageReceipt(db, envelopeSchema.parse(raw));
-    await fs.rm(file, { force: true });
+    if (!moved) return [...recovered];
   }
+  throw new Error("Accounting spool changed during recovery; retry required");
 }
 
 export async function createRunUsageRecorder(db: Db, input: { companyId: string; runId: string; adapterType: string }, directory = usageReceiptSpoolPath()) {