fix(heartbeat): retry a legacy Stop that loses the task-row lock race (#21492)

Legacy Stop retries its terminal write up to 3 times when another writer
holds the task row past the 1s lock_timeout, and returns a retryable
409 run_stop_busy instead of a 500 when contention persists. Board Stop
also bounds its earlier applySupersession task-row lock to 1s so it can
no longer wait without limit.

Co-Authored-By: Paperclip <paperclip@users.noreply.github.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
authored and GitHub committed 2026-11-13 14:29:44 -08:00
1 parent 999709b0a6
commit 4c5e7bf867
3 files changed
+118 -5

No files matched your search

@@ -24,6 +24,7 @@ import { truncateTablesWithDeadlockRetry } from "./helpers/truncate-with-deadloc
import { heartbeatService } from "../services/heartbeat.ts";
import { returnStoppedWaitingTaskToTodo } from "../services/heartbeat/stopped-waiting-task.ts";
import type { AdapterExecutionContext, AdapterExecutionResult } from "../adapters/index.ts";
import { HttpError } from "../errors.ts";
const gates = vi.hoisted(() => new Map<string, (() => void) | null>());
const mockAdapterExecute = vi.hoisted(() =>
@@ -149,4 +150,79 @@ describeEmbeddedPostgres("Stop on a task whose run is still waiting in line", ()
expect(afterIssue.executionRunId).toBeNull();
expect(busyRun).toBeTruthy();
}, 120_000);
/** Hold the task row in another transaction until `release` is called. */
async function holdTaskRow(issueId: string) {
let release!: () => void;
const released = new Promise<void>((resolve) => { release = resolve; });
let locked!: () => void;
const lockTaken = new Promise<void>((resolve) => { locked = resolve; });
const holder = db.transaction(async (tx) => {
await tx.select({ id: issues.id }).from(issues).where(eq(issues.id, issueId)).for("update");
locked();
await released;
});
await lockTaken;
return { release, holder };
}
async function queueBehindBusyTask() {
const { companyId, agentId } = await seed();
const [busyIssue] = await db.insert(issues).values({ companyId, title: "Busy", status: "todo", assigneeAgentId: agentId }).returning();
gates.set(busyIssue.id, null);
await heartbeat.wakeup(agentId, { source: "assignment", triggerDetail: "system", reason: "issue_assigned",
requestedByActorType: "system", payload: { issueId: busyIssue.id } });
await vi.waitFor(() => expect(executedIssueIds()).toContain(busyIssue.id), { timeout: 10_000 });
const waiting = await assignAndWake(companyId, agentId, "Waiting task");
const queued = await heartbeat.getRun(waiting.run.id);
expect(queued?.status).toBe("queued");
expect(queued?.runtimeMode).toBe("legacy");
return waiting;
}
// The board route attributes Stop to the user, which first supersedes the
// task's handoffs under the same task-row lock.
const stopCallers = [
{ caller: "control plane", reason: "Cancelled by control plane", options: undefined },
{ caller: "board", reason: "Cancelled by a board operator",
options: { resultJson: { cancelledByActorType: "user", cancelledByUserId: "responsible-user" } } },
] as const;
it.each(stopCallers)("stops a waiting legacy run for the $caller when another writer briefly holds its task row past the lock timeout", async ({ reason, options }) => {
const waiting = await queueBehindBusyTask();
const lock = await holdTaskRow(waiting.issue.id);
// Longer than one 1s lock_timeout, shorter than the bounded attempts.
const releaseTimer = setTimeout(lock.release, 1_500);
try {
const stopped = await heartbeat.cancelRun(waiting.run.id, reason, options);
expect(stopped?.status).toBe("cancelled");
} finally {
clearTimeout(releaseTimer);
lock.release();
await lock.holder;
}
expect((await heartbeat.getRun(waiting.run.id))?.status).toBe("cancelled");
}, 60_000);
it.each(stopCallers)("answers the $caller a retryable conflict when the task row stays locked through every attempt", async ({ reason, options }) => {
const waiting = await queueBehindBusyTask();
const lock = await holdTaskRow(waiting.issue.id);
try {
const startedAt = Date.now();
const error = await heartbeat.cancelRun(waiting.run.id, reason, options).then(
() => null, (caught: unknown) => caught);
expect(error).toBeInstanceOf(HttpError);
expect((error as HttpError).status).toBe(409);
expect((error as HttpError).details).toMatchObject({ code: "run_stop_busy", runId: waiting.run.id });
// Bounded: three 1s waits plus short backoff, never an open-ended queue.
expect(Date.now() - startedAt).toBeLessThan(10_000);
} finally {
lock.release();
await lock.holder;
}
// Nothing was committed, so the same Stop succeeds once the writer is done.
expect((await heartbeat.getRun(waiting.run.id))?.status).toBe("queued");
const stopped = await heartbeat.cancelRun(waiting.run.id, reason, options);
expect(stopped?.status).toBe("cancelled");
}, 60_000);
});
+29 -3
View File
@@ -72,6 +72,7 @@ import {
waitForAdapterStop,
} from "../adapter-execution-control.js";
import { AdapterStopTimeoutError } from "../adapter-stop-timeout.js";
import { isTransientLockContention, postgresErrorCode } from "../transient-lock-contention.js";
import { pendingBudgetRunCancellationPredicate, pendingBudgetRunIdsForScopeQuery } from "./budget-cancellation.js";
import { notifyBudgetStopsSettled } from "../budget-resume-continuation.js";
import { withBudgetSpan } from "../budget-tracing.js";
@@ -1249,10 +1250,19 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
let caseHandoffRetryClaimed = false;
if (handoffIssueId && supersedesCaseHandoff && options.resultJson?.caseHandoffYieldStop !== true) {
const publications: ActivityPublication[] = [];
let taskRowLocked = false;
const applySupersession = () => db.transaction(async tx => {
taskRowLocked = false;
// As in legacy terminalization, wait at most 1s for the task row so a
// board Stop never queues behind a slow task writer. Later locks keep
// the transaction's previous timeout.
const [settings] = await tx.execute(sql`select current_setting('lock_timeout') as lock_timeout`);
await tx.execute(sql`select set_config('lock_timeout', '1000', true)`);
const [issue] = await tx.select({ id: issues.id, executionRunId: issues.executionRunId, checkoutRunId: issues.checkoutRunId,
assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId }).from(issues)
.where(and(eq(issues.companyId, run!.companyId), eq(issues.id, handoffIssueId))).for("update");
taskRowLocked = true;
await tx.execute(sql`select set_config('lock_timeout', ${String(settings.lock_timeout)}, true)`);
const [current] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, runId),
eq(heartbeatRuns.companyId, run!.companyId))).for("update");
if (!current || (options.expectedStatus && current.status !== options.expectedStatus)) return false;
@@ -1313,9 +1323,18 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
supersessionApplied = await applySupersession();
break;
} catch (error) {
const value = parseObject(error);
const deadlock = value.code === "40P01" || parseObject(value.cause).code === "40P01";
if (!deadlock || attempt >= 2) rethrowNativeCancellationLockConflict(error);
const code = postgresErrorCode(error);
const taskRowBusy = code === "55P03" && !taskRowLocked;
if ((code !== "40P01" && !taskRowBusy) || attempt >= 2) {
// The task row stayed locked through every attempt. Nothing was
// committed, so the same Stop can be sent again.
if (taskRowBusy) {
throw conflict("The task is busy, so Stop could not finish. Try Stop again.", {
code: "run_stop_busy", runId,
});
}
rethrowNativeCancellationLockConflict(error);
}
// PostgreSQL aborted the entire transaction. Discard its local
// receipts before re-reading current authority under fresh locks.
// Provider Stop and publication happen only after this boundary.
@@ -1646,6 +1665,13 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
);
}
}
// Another writer held the task row through every bounded attempt.
// The terminal write rolled back, so the same Stop can be sent again.
if (isTransientLockContention(error)) {
throw conflict("The task is busy, so Stop could not finish. Try Stop again.", {
code: "run_stop_busy", runId: run.id,
});
}
throw error;
} finally {
if (processCancellationSettlement) {
+13 -2
View File
@@ -59,6 +59,7 @@ import {
isConversationAdapter,
} from "../conversation-continuation.js";
import { terminalizeLegacyExecution } from "../legacy-execution-recovery.js";
import { retryTransientLockContention } from "../transient-lock-contention.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
import { buildHeartbeatRunStatusLiveEventPayload } from "../heartbeat-run-status-payload.js";
import { randomUUID } from "node:crypto";
@@ -289,6 +290,9 @@ const PRE_ADAPTER_SETUP_FAILURE_CODES = new Set<string>([
...NON_RETRYABLE_PREFLIGHT_FAILURE_CODES,
]);
/** Each attempt waits at most the 1s task-row lock_timeout, so a Stop stays prompt. */
const LEGACY_TERMINALIZATION_LOCK_ATTEMPTS = 3;
const MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS = 2;
const MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP = 10;
@@ -1279,7 +1283,8 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl
},
};
}
const updated = await writeRunStatus(status, previousStatus, async writer => {
const legacyTerminalization = previousStatus?.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(status);
const write = () => writeRunStatus(status, previousStatus, async writer => {
if (options?.setupFailure && previousStatus?.runtimeMode === "native" && previousStatus.nativeIssueId) {
await writer.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations)
.where(and(eq(nativeRunFinalizations.runId, runId),
@@ -1292,7 +1297,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl
eq(nativeRunFinalizations.companyId, previousStatus.companyId)))
.for("update");
}
return previousStatus && previousStatus.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(status)
return previousStatus && legacyTerminalization
? terminalizeLegacyExecution({
db: writer,
run: previousStatus,
@@ -1347,6 +1352,12 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl
throw error;
});
});
// Legacy terminalization takes the task row under a short lock_timeout so
// it never queues behind a slow task writer. Losing that race rolls back
// the whole transaction, so a bounded fresh attempt is safe.
const updated = legacyTerminalization
? await retryTransientLockContention(write, LEGACY_TERMINALIZATION_LOCK_ATTEMPTS)
: await write();
if (updated) {
publishLiveEvent({