From 7747487ee766c9e28aa06cad758ad4a4eb08e3ab Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:33:36 -0600 Subject: [PATCH] fix(budgets): retry contended accounting writes with jittered backoff (#21185) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Budget policies and usage records are written in serialized accounting transactions for each company > - When several writes for one company run at the same time, PostgreSQL rejects all but one and Paperclip retries the others > - The retries were few and short (4 attempts, 10 to 30 ms, the same delay for every writer), so concurrent writers collided again and the request failed with 503 > - The 503 message also said "this run's budget" for a policy write that has no run > - This pull request retries contended accounting writes with jittered exponential backoff and gives the error a correct message > - The benefit is that normal concurrent budget changes succeed, and a real failure tells the user what happened ## Linked Issues or Issue Description **What happened?** Several budget policy writes for one company at the same time returned 503 after 4 serialization retries. The same retry exhaustion appeared often in server logs of the chat feature evals. The message said "Paperclip could not save this run's budget or usage records … Retry the run" for a policy write. **Expected behavior** Concurrent budget policy writes succeed. If contention continues, the error describes budget or usage changes, not a run. **Steps to reproduce** Send 9 budget policy upserts for one company at the same time. Before this change, about half of them fail with 503. **Paperclip version or commit** master ## What Changed - `withOperationAccountingTransaction` retries up to `ACCOUNTING_TRANSACTION_ATTEMPTS` (8) times. The delay before each retry is jittered exponential backoff (`accountingRetryDelayMs`): 10 ms × 2^(attempt+1), capped at 400 ms, randomized between half and the full value. The worst case total wait is about 1.4 s. - `AccountingContentionError` uses a new generic message: "Paperclip could not save these budget or usage changes because other updates to the same company kept conflicting with them. Try again." - A failed run keeps its run-specific `ACCOUNTING_CONTENTION_MESSAGE` (run completion still uses it). - The existing exhausted-retry test uses the new attempt constant. ## Verification - `cd server && npx vitest run src/__tests__/budget-policy-write-contention.test.ts` - Red before the change: 5 of 9 concurrent policy writes failed with `AccountingContentionError`, and the message mentioned a run. - Green after: all writes succeed (5 of 5 repeated runs; also 3 of 3 runs with 17 concurrent writers). - `run-accounting-contention`, `run-completion`, `budget-control-publication`, `budget-evaluation-work` and `heartbeat-native-predispatch-retry` tests. - Paid runner E2E evidence for `goals.align-existing-work` is linked in a comment. ## Risks - Low. A write that keeps losing now waits up to about 1.4 s before it fails, instead of about 60 ms. Only transactions that PostgreSQL rolled back (40001 or 40P01) are retried, as before. - Moving policy writes out of the accounting transaction was considered. It is not done here, because the policy fence keeps budget evaluation consistent with policy changes. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Anthropic Claude Opus 5.5 (`claude-opus-5-5`) in Claude Code, with tool use and code execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../budget-policy-write-contention.test.ts | 64 +++++++++++++++++++ server/src/services/accounting-transaction.ts | 19 +++++- .../run-accounting-contention.test.ts | 6 +- 3 files changed, 83 insertions(+), 6 deletions(-) create mode 100644 server/src/__tests__/budget-policy-write-contention.test.ts diff --git a/server/src/__tests__/budget-policy-write-contention.test.ts b/server/src/__tests__/budget-policy-write-contention.test.ts new file mode 100644 index 00000000000..f72052bef44 --- /dev/null +++ b/server/src/__tests__/budget-policy-write-contention.test.ts @@ -0,0 +1,64 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, budgetPolicies, companies, createDb } from "@paperclipai/db"; +import { AccountingContentionError, accountingRetryDelayMs } from "../services/accounting-transaction.js"; +import { budgetService } from "../services/budgets.js"; +import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +describe("accounting contention message", () => { + it("describes the conflicting write without blaming a run", () => { + const error = new AccountingContentionError(Object.assign(new Error("could not serialize"), { code: "40001" }), 8); + expect(error.status).toBe(503); + expect(error.message).not.toMatch(/\brun\b/i); + expect(error.message).toMatch(/budget/i); + }); +}); + +describe("accounting retry delay", () => { + it("doubles from 20 ms, randomizes between half and the full ceiling, and caps at 400 ms", () => { + const low = () => 0; + const high = () => 1; + expect([0, 1, 2, 3, 4, 5, 6].map((attempt) => accountingRetryDelayMs(attempt, high))) + .toEqual([20, 40, 80, 160, 320, 400, 400]); + expect([0, 1, 2, 3, 4, 5, 6].map((attempt) => accountingRetryDelayMs(attempt, low))) + .toEqual([10, 20, 40, 80, 160, 200, 200]); + expect(accountingRetryDelayMs(2, () => 0.5)).toBe(60); + }); +}); + +const support = await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("concurrent budget policy writes", () => { + let temp: Awaited>; + let db: ReturnType; + beforeAll(async () => { + temp = await startEmbeddedPostgresTestDatabase("paperclip-budget-policy-contention-"); + db = createDb(temp.connectionString); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterAll(async () => { + await db?.$client.end({ timeout: 1 }); + await temp?.cleanup(); + }); + + it("saves every policy when several are written for one company at once", async () => { + const companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Budget writes", issuePrefix: `B${companyId.slice(0, 6)}` }); + const agentIds = Array.from({ length: 8 }, () => randomUUID()); + await db.insert(agents).values(agentIds.map((id, index) => ({ + id, companyId, name: `Worker ${index}`, role: "engineer", status: "active" as const, adapterType: "process", + }))); + const budgets = budgetService(db); + await budgets.upsertPolicy(companyId, { scopeType: "company", scopeId: companyId, amount: 1000 }, null); + + const results = await Promise.allSettled([ + ...agentIds.map((agentId, index) => budgets.upsertPolicy(companyId, + { scopeType: "agent", scopeId: agentId, amount: 100 + index }, null)), + budgets.upsertPolicy(companyId, { scopeType: "company", scopeId: companyId, amount: 2000 }, null), + ]); + + expect(results.filter(result => result.status === "rejected")).toEqual([]); + const saved = await db.select().from(budgetPolicies).where(eq(budgetPolicies.companyId, companyId)); + expect(saved).toHaveLength(agentIds.length + 1); + expect(saved.find(policy => policy.scopeType === "company")?.amount).toBe(2000); + }, 60_000); +}); diff --git a/server/src/services/accounting-transaction.ts b/server/src/services/accounting-transaction.ts index e87eef79aa0..dd3f374b722 100644 --- a/server/src/services/accounting-transaction.ts +++ b/server/src/services/accounting-transaction.ts @@ -43,27 +43,40 @@ export async function withOperationAccountingTransaction(db: Db, companyId: s return result; } catch (error) { if (!isAccountingRetryable(error)) throw error; - if (attempt >= 3) { + if (attempt + 1 >= ACCOUNTING_TRANSACTION_ATTEMPTS) { const contention = new AccountingContentionError(error, attempt + 1); logger.warn({ companyId, sqlState: contention.code, attempts: contention.attempts, err: error }, "Accounting transaction exhausted its serialization retries"); throw contention; } - await new Promise(resolve => setTimeout(resolve, 10 * (attempt + 1))); + await new Promise(resolve => setTimeout(resolve, accountingRetryDelayMs(attempt))); } } } +export const ACCOUNTING_TRANSACTION_ATTEMPTS = 8; + +const ACCOUNTING_RETRY_BASE_DELAY_MS = 10; +const ACCOUNTING_RETRY_MAX_DELAY_MS = 400; + +export function accountingRetryDelayMs(attempt: number, random: () => number = Math.random): number { + const ceiling = Math.min(ACCOUNTING_RETRY_MAX_DELAY_MS, ACCOUNTING_RETRY_BASE_DELAY_MS * 2 ** (attempt + 1)); + return Math.round(ceiling / 2 + random() * (ceiling / 2)); +} + export const ACCOUNTING_CONTENTION_MESSAGE = "Paperclip could not save this run's budget or usage records because other " + "updates to the same run kept conflicting with them. Retry the run."; +export const ACCOUNTING_WRITE_CONTENTION_MESSAGE = "Paperclip could not save these budget or usage changes because other " + + "updates to the same company kept conflicting with them. Try again."; + /** Bounded, SQL-free failure for exhausted serialization retries. `code` * keeps the SQLSTATE (40001 or 40P01) for retry classifiers and diagnostics; * the database error, with its statement text, stays only in `cause`. */ export class AccountingContentionError extends HttpError { readonly code: string; constructor(cause: unknown, readonly attempts: number) { - super(503, ACCOUNTING_CONTENTION_MESSAGE); + super(503, ACCOUNTING_WRITE_CONTENTION_MESSAGE); this.name = "AccountingContentionError"; this.code = accountingRetryCode(cause) ?? "40001"; this.cause = cause; diff --git a/server/src/services/run-accounting-contention.test.ts b/server/src/services/run-accounting-contention.test.ts index 6a9e4ef2b10..54eccaa7a97 100644 --- a/server/src/services/run-accounting-contention.test.ts +++ b/server/src/services/run-accounting-contention.test.ts @@ -20,7 +20,7 @@ import { } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { - AccountingContentionError, isAccountingContentionError, withOperationAccountingTransaction, + ACCOUNTING_TRANSACTION_ATTEMPTS, AccountingContentionError, isAccountingContentionError, withOperationAccountingTransaction, } from "./accounting-transaction.js"; import { createPostgresRunDispatchAdapter } from "../modules/run-dispatch/adapters/postgres.js"; import { reserveRunBudget } from "./budget-reservations.js"; @@ -290,11 +290,11 @@ describe.skipIf(!support.supported)("run accounting writers under concurrent run const error = await withOperationAccountingTransaction(hooked.db, f.companyId, async tx => { await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).for("update"); }).then(() => null, (failure: unknown) => failure); - expect(hooked.interleavings()).toBe(4); + expect(hooked.interleavings()).toBe(ACCOUNTING_TRANSACTION_ATTEMPTS); expect(error).toBeInstanceOf(AccountingContentionError); expect(isAccountingContentionError(error)).toBe(true); const contention = error as AccountingContentionError; - expect(contention).toMatchObject({ code: "40001", attempts: 4, status: 503 }); + expect(contention).toMatchObject({ code: "40001", attempts: ACCOUNTING_TRANSACTION_ATTEMPTS, status: 503 }); expect(contention.message).not.toMatch(/select|heartbeat_runs|Failed query|params/i); // The database error stays attached for the server log only. expect(String((contention.cause as Error).message)).toMatch(/^Failed query: select/);