From a0979498642a2779e5a1a0f758db62a3580f9301 Mon Sep 17 00:00:00 2001 From: paperclipbugbot Date: Fri, 20 Nov 2026 06:17:00 -0800 Subject: [PATCH] fix(server): stop issue reference syncs from deadlocking on mutual mentions (#22389) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The server keeps a table of issue reference mentions. It rebuilds the rows for an issue when its title or description changes. > - `syncIssue` locks the source issue row `FOR UPDATE` before it rebuilds the rows (added in #13898). > - Each mention insert does a foreign-key check. That check takes a `FOR KEY SHARE` lock on the target issue. `FOR KEY SHARE` conflicts with `FOR UPDATE`. > - When issue A mentions B and issue B mentions A, two concurrent syncs each hold one `FOR UPDATE` lock and wait for the other. Postgres reports `deadlock detected`, and one update fails. We saw this on staging. > - This pull request changes the lock to `FOR NO KEY UPDATE`. > - The benefit is that syncs of one issue stay serialized, and foreign-key checks from other issues do not wait on that lock. ## Linked Issues or Issue Description Refs #13898 **What happened?** A staging server reported `Failed query: insert into "issue_reference_mentions" ...` caused by `PostgresError: deadlock detected`. The insert came from `replaceSourceMentions` inside the `syncIssue` transaction. The source issue description mentioned the target issue. **Steps to reproduce** 1. Create issue A with a description that mentions issue B. 2. Create issue B with a description that mentions issue A. 3. Run `syncIssue(A)` and `syncIssue(B)` at the same time, several times. 4. One sync fails with `deadlock detected`. **Expected behavior** Both syncs complete, and each issue has its mention rows. ## What Changed - `server/src/services/issue-references.ts`: `syncIssue` now locks the source issue with `FOR NO KEY UPDATE`, not `FOR UPDATE`. `FOR NO KEY UPDATE` still conflicts with itself, so concurrent syncs of one issue stay serialized (the #13898 behavior). It does not conflict with the `FOR KEY SHARE` lock of a foreign-key check, so the lock cycle cannot form. - `server/src/__tests__/issue-references-service.test.ts`: two regression tests. - One test holds a sync lock in an open transaction. It then syncs a second issue that mentions the locked issue, and expects that sync to complete without a wait. - One test runs 20 concurrent syncs of two issues that mention each other, and expects all four mention rows. ## Verification - `cd server && npx vitest run src/__tests__/issue-references-service.test.ts` - Before the fix (old lock): the two new tests fail. One test shows `expected 'blocked' to be 'completed'`. The other test fails with `PostgresError: deadlock detected` on `select ... from "issues" ... for update`. This is the same error as the staging event. - After the fix: 5/5 tests pass. I ran the file three times and all runs passed. The existing test "serializes concurrent syncs for a description with repeated references" still passes. ## Risks - Low risk. There is no schema change or migration. The lock mode changes only for this read. `UPDATE` statements that do not change key columns already take `FOR NO KEY UPDATE`, so this lock blocks the same issue writers as before. Only foreign-key `KEY SHARE` checks can now continue at the same time. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), with tool use and code execution in a Claude Code agent session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: paperclipbugbot <356415228+paperclipbugbot@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- .../issue-references-service.test.ts | 104 ++++++++++++++++++ server/src/services/issue-references.ts | 2 +- 2 files changed, 105 insertions(+), 1 deletion(-) diff --git a/server/src/__tests__/issue-references-service.test.ts b/server/src/__tests__/issue-references-service.test.ts index 3954b5dd5b8..b98ea19b604 100644 --- a/server/src/__tests__/issue-references-service.test.ts +++ b/server/src/__tests__/issue-references-service.test.ts @@ -218,6 +218,110 @@ describeEmbeddedPostgres("issueReferenceService", () => { expect(mentions[0]?.targetIssueId).toBe(targetIssueId); }); + it("does not block mention foreign-key checks while an issue sync holds its lock", async () => { + const companyId = randomUUID(); + const heldIssueId = randomUUID(); + const mentioningIssueId = randomUUID(); + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip Reference Locks", + issuePrefix: `L${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + requireBoardApprovalForNewAgents: false, + }); + await db.insert(issues).values([ + { + id: heldIssueId, + companyId, + title: "Held issue", + description: "Depends on LCK-2.", + status: "todo", + priority: "medium", + identifier: "LCK-1", + }, + { + id: mentioningIssueId, + companyId, + title: "Mentioning issue", + description: "Blocked by LCK-1.", + status: "todo", + priority: "medium", + identifier: "LCK-2", + }, + ]); + + let releaseHolder!: () => void; + const holderReleased = new Promise((resolve) => { releaseHolder = resolve; }); + let markHolderLocked!: () => void; + const holderLocked = new Promise((resolve) => { markHolderLocked = resolve; }); + const holder = db.transaction(async (tx) => { + await refs.syncIssue(heldIssueId, tx); + markHolderLocked(); + await holderReleased; + }); + + try { + await holderLocked; + const otherSync = refs.syncIssue(mentioningIssueId); + const outcome = await Promise.race([ + otherSync.then(() => "completed" as const), + new Promise<"blocked">((resolve) => setTimeout(() => resolve("blocked"), 1_500)), + ]); + releaseHolder(); + await otherSync; + expect(outcome).toBe("completed"); + } finally { + releaseHolder(); + await holder; + } + + const mentions = await db.select().from(issueReferenceMentions); + expect(mentions.map((row) => `${row.sourceIssueId}->${row.targetIssueId}`).sort()).toEqual([ + `${heldIssueId}->${mentioningIssueId}`, + `${mentioningIssueId}->${heldIssueId}`, + ].sort()); + }); + + it("syncs issues that mention each other concurrently without deadlocking", async () => { + const companyId = randomUUID(); + const firstIssueId = randomUUID(); + const secondIssueId = randomUUID(); + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip Mutual References", + issuePrefix: `M${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + requireBoardApprovalForNewAgents: false, + }); + await db.insert(issues).values([ + { + id: firstIssueId, + companyId, + title: "Parent MUT-2", + description: "Track MUT-2 until it lands.", + status: "todo", + priority: "medium", + identifier: "MUT-1", + }, + { + id: secondIssueId, + companyId, + title: "Child of MUT-1", + description: "Reports back to MUT-1.", + status: "todo", + priority: "medium", + identifier: "MUT-2", + }, + ]); + + await expect(Promise.all( + Array.from({ length: 10 }, () => [refs.syncIssue(firstIssueId), refs.syncIssue(secondIssueId)]).flat(), + )).resolves.toHaveLength(20); + + const mentions = await db.select().from(issueReferenceMentions); + expect(mentions).toHaveLength(4); + }); + it("backfills existing references for a company without requiring write-time sync", async () => { const companyId = randomUUID(); const sourceIssueId = randomUUID(); diff --git a/server/src/services/issue-references.ts b/server/src/services/issue-references.ts index c66facd6451..4a8f9a3c15c 100644 --- a/server/src/services/issue-references.ts +++ b/server/src/services/issue-references.ts @@ -184,7 +184,7 @@ export function issueReferenceService(db: Db) { }) .from(issues) .where(eq(issues.id, issueId)); - return (lock ? query.for("update") : query) + return (lock ? query.for("no key update") : query) .then((rows: Array<{ id: string; companyId: string; title: string; description: string | null }>) => rows[0] ?? null); }