From 4c5e7bf8676057bf77e3bfe2df8126b6ae4e501b Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Fri, 13 Nov 2026 14:29:44 -0800 Subject: [PATCH] fix(heartbeat): retry a legacy Stop that loses the task-row lock race (#21492) Legacy Stop retries its terminal write up to 3 times when another writer holds the task row past the 1s lock_timeout, and returns a retryable 409 run_stop_busy instead of a 500 when contention persists. Board Stop also bounds its earlier applySupersession task-row lock to 1s so it can no longer wait without limit. Co-Authored-By: Paperclip Co-Authored-By: Claude Opus 5.5 --- .../heartbeat-stop-queued-task.test.ts | 76 +++++++++++++++++++ server/src/services/heartbeat/run-control.ts | 32 +++++++- .../src/services/heartbeat/run-lifecycle.ts | 15 +++- 3 files changed, 118 insertions(+), 5 deletions(-) diff --git a/server/src/__tests__/heartbeat-stop-queued-task.test.ts b/server/src/__tests__/heartbeat-stop-queued-task.test.ts index f99b5537d86..56fcc97aa87 100644 --- a/server/src/__tests__/heartbeat-stop-queued-task.test.ts +++ b/server/src/__tests__/heartbeat-stop-queued-task.test.ts @@ -24,6 +24,7 @@ import { truncateTablesWithDeadlockRetry } from "./helpers/truncate-with-deadloc import { heartbeatService } from "../services/heartbeat.ts"; import { returnStoppedWaitingTaskToTodo } from "../services/heartbeat/stopped-waiting-task.ts"; import type { AdapterExecutionContext, AdapterExecutionResult } from "../adapters/index.ts"; +import { HttpError } from "../errors.ts"; const gates = vi.hoisted(() => new Map void) | null>()); const mockAdapterExecute = vi.hoisted(() => @@ -149,4 +150,79 @@ describeEmbeddedPostgres("Stop on a task whose run is still waiting in line", () expect(afterIssue.executionRunId).toBeNull(); expect(busyRun).toBeTruthy(); }, 120_000); + + /** Hold the task row in another transaction until `release` is called. */ + async function holdTaskRow(issueId: string) { + let release!: () => void; + const released = new Promise((resolve) => { release = resolve; }); + let locked!: () => void; + const lockTaken = new Promise((resolve) => { locked = resolve; }); + const holder = db.transaction(async (tx) => { + await tx.select({ id: issues.id }).from(issues).where(eq(issues.id, issueId)).for("update"); + locked(); + await released; + }); + await lockTaken; + return { release, holder }; + } + + async function queueBehindBusyTask() { + const { companyId, agentId } = await seed(); + const [busyIssue] = await db.insert(issues).values({ companyId, title: "Busy", status: "todo", assigneeAgentId: agentId }).returning(); + gates.set(busyIssue.id, null); + await heartbeat.wakeup(agentId, { source: "assignment", triggerDetail: "system", reason: "issue_assigned", + requestedByActorType: "system", payload: { issueId: busyIssue.id } }); + await vi.waitFor(() => expect(executedIssueIds()).toContain(busyIssue.id), { timeout: 10_000 }); + const waiting = await assignAndWake(companyId, agentId, "Waiting task"); + const queued = await heartbeat.getRun(waiting.run.id); + expect(queued?.status).toBe("queued"); + expect(queued?.runtimeMode).toBe("legacy"); + return waiting; + } + + // The board route attributes Stop to the user, which first supersedes the + // task's handoffs under the same task-row lock. + const stopCallers = [ + { caller: "control plane", reason: "Cancelled by control plane", options: undefined }, + { caller: "board", reason: "Cancelled by a board operator", + options: { resultJson: { cancelledByActorType: "user", cancelledByUserId: "responsible-user" } } }, + ] as const; + + it.each(stopCallers)("stops a waiting legacy run for the $caller when another writer briefly holds its task row past the lock timeout", async ({ reason, options }) => { + const waiting = await queueBehindBusyTask(); + const lock = await holdTaskRow(waiting.issue.id); + // Longer than one 1s lock_timeout, shorter than the bounded attempts. + const releaseTimer = setTimeout(lock.release, 1_500); + try { + const stopped = await heartbeat.cancelRun(waiting.run.id, reason, options); + expect(stopped?.status).toBe("cancelled"); + } finally { + clearTimeout(releaseTimer); + lock.release(); + await lock.holder; + } + expect((await heartbeat.getRun(waiting.run.id))?.status).toBe("cancelled"); + }, 60_000); + + it.each(stopCallers)("answers the $caller a retryable conflict when the task row stays locked through every attempt", async ({ reason, options }) => { + const waiting = await queueBehindBusyTask(); + const lock = await holdTaskRow(waiting.issue.id); + try { + const startedAt = Date.now(); + const error = await heartbeat.cancelRun(waiting.run.id, reason, options).then( + () => null, (caught: unknown) => caught); + expect(error).toBeInstanceOf(HttpError); + expect((error as HttpError).status).toBe(409); + expect((error as HttpError).details).toMatchObject({ code: "run_stop_busy", runId: waiting.run.id }); + // Bounded: three 1s waits plus short backoff, never an open-ended queue. + expect(Date.now() - startedAt).toBeLessThan(10_000); + } finally { + lock.release(); + await lock.holder; + } + // Nothing was committed, so the same Stop succeeds once the writer is done. + expect((await heartbeat.getRun(waiting.run.id))?.status).toBe("queued"); + const stopped = await heartbeat.cancelRun(waiting.run.id, reason, options); + expect(stopped?.status).toBe("cancelled"); + }, 60_000); }); diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts index f77e8f9b013..ccc2a21b872 100644 --- a/server/src/services/heartbeat/run-control.ts +++ b/server/src/services/heartbeat/run-control.ts @@ -72,6 +72,7 @@ import { waitForAdapterStop, } from "../adapter-execution-control.js"; import { AdapterStopTimeoutError } from "../adapter-stop-timeout.js"; +import { isTransientLockContention, postgresErrorCode } from "../transient-lock-contention.js"; import { pendingBudgetRunCancellationPredicate, pendingBudgetRunIdsForScopeQuery } from "./budget-cancellation.js"; import { notifyBudgetStopsSettled } from "../budget-resume-continuation.js"; import { withBudgetSpan } from "../budget-tracing.js"; @@ -1249,10 +1250,19 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont let caseHandoffRetryClaimed = false; if (handoffIssueId && supersedesCaseHandoff && options.resultJson?.caseHandoffYieldStop !== true) { const publications: ActivityPublication[] = []; + let taskRowLocked = false; const applySupersession = () => db.transaction(async tx => { + taskRowLocked = false; + // As in legacy terminalization, wait at most 1s for the task row so a + // board Stop never queues behind a slow task writer. Later locks keep + // the transaction's previous timeout. + const [settings] = await tx.execute(sql`select current_setting('lock_timeout') as lock_timeout`); + await tx.execute(sql`select set_config('lock_timeout', '1000', true)`); const [issue] = await tx.select({ id: issues.id, executionRunId: issues.executionRunId, checkoutRunId: issues.checkoutRunId, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId }).from(issues) .where(and(eq(issues.companyId, run!.companyId), eq(issues.id, handoffIssueId))).for("update"); + taskRowLocked = true; + await tx.execute(sql`select set_config('lock_timeout', ${String(settings.lock_timeout)}, true)`); const [current] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, run!.companyId))).for("update"); if (!current || (options.expectedStatus && current.status !== options.expectedStatus)) return false; @@ -1313,9 +1323,18 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont supersessionApplied = await applySupersession(); break; } catch (error) { - const value = parseObject(error); - const deadlock = value.code === "40P01" || parseObject(value.cause).code === "40P01"; - if (!deadlock || attempt >= 2) rethrowNativeCancellationLockConflict(error); + const code = postgresErrorCode(error); + const taskRowBusy = code === "55P03" && !taskRowLocked; + if ((code !== "40P01" && !taskRowBusy) || attempt >= 2) { + // The task row stayed locked through every attempt. Nothing was + // committed, so the same Stop can be sent again. + if (taskRowBusy) { + throw conflict("The task is busy, so Stop could not finish. Try Stop again.", { + code: "run_stop_busy", runId, + }); + } + rethrowNativeCancellationLockConflict(error); + } // PostgreSQL aborted the entire transaction. Discard its local // receipts before re-reading current authority under fresh locks. // Provider Stop and publication happen only after this boundary. @@ -1646,6 +1665,13 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont ); } } + // Another writer held the task row through every bounded attempt. + // The terminal write rolled back, so the same Stop can be sent again. + if (isTransientLockContention(error)) { + throw conflict("The task is busy, so Stop could not finish. Try Stop again.", { + code: "run_stop_busy", runId: run.id, + }); + } throw error; } finally { if (processCancellationSettlement) { diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index 77e1030f039..fa8ea45b717 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -59,6 +59,7 @@ import { isConversationAdapter, } from "../conversation-continuation.js"; import { terminalizeLegacyExecution } from "../legacy-execution-recovery.js"; +import { retryTransientLockContention } from "../transient-lock-contention.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { buildHeartbeatRunStatusLiveEventPayload } from "../heartbeat-run-status-payload.js"; import { randomUUID } from "node:crypto"; @@ -289,6 +290,9 @@ const PRE_ADAPTER_SETUP_FAILURE_CODES = new Set([ ...NON_RETRYABLE_PREFLIGHT_FAILURE_CODES, ]); +/** Each attempt waits at most the 1s task-row lock_timeout, so a Stop stays prompt. */ +const LEGACY_TERMINALIZATION_LOCK_ATTEMPTS = 3; + const MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS = 2; const MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP = 10; @@ -1279,7 +1283,8 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl }, }; } - const updated = await writeRunStatus(status, previousStatus, async writer => { + const legacyTerminalization = previousStatus?.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(status); + const write = () => writeRunStatus(status, previousStatus, async writer => { if (options?.setupFailure && previousStatus?.runtimeMode === "native" && previousStatus.nativeIssueId) { await writer.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations) .where(and(eq(nativeRunFinalizations.runId, runId), @@ -1292,7 +1297,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl eq(nativeRunFinalizations.companyId, previousStatus.companyId))) .for("update"); } - return previousStatus && previousStatus.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(status) + return previousStatus && legacyTerminalization ? terminalizeLegacyExecution({ db: writer, run: previousStatus, @@ -1347,6 +1352,12 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl throw error; }); }); + // Legacy terminalization takes the task row under a short lock_timeout so + // it never queues behind a slow task writer. Losing that race rolls back + // the whole transaction, so a bounded fresh attempt is safe. + const updated = legacyTerminalization + ? await retryTransientLockContention(write, LEGACY_TERMINALIZATION_LOCK_ATTEMPTS) + : await write(); if (updated) { publishLiveEvent({