mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 03:31:44 +01:00
fix(heartbeat): retry a legacy Stop that loses the task-row lock race (#21492)
Legacy Stop retries its terminal write up to 3 times when another writer holds the task row past the 1s lock_timeout, and returns a retryable 409 run_stop_busy instead of a 500 when contention persists. Board Stop also bounds its earlier applySupersession task-row lock to 1s so it can no longer wait without limit. Co-Authored-By: Paperclip <paperclip@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
3 files changed
+118
-5
No files matched your search
@@ -24,6 +24,7 @@ import { truncateTablesWithDeadlockRetry } from "./helpers/truncate-with-deadloc
|
||||
import { heartbeatService } from "../services/heartbeat.ts";
|
||||
import { returnStoppedWaitingTaskToTodo } from "../services/heartbeat/stopped-waiting-task.ts";
|
||||
import type { AdapterExecutionContext, AdapterExecutionResult } from "../adapters/index.ts";
|
||||
import { HttpError } from "../errors.ts";
|
||||
|
||||
const gates = vi.hoisted(() => new Map<string, (() => void) | null>());
|
||||
const mockAdapterExecute = vi.hoisted(() =>
|
||||
@@ -149,4 +150,79 @@ describeEmbeddedPostgres("Stop on a task whose run is still waiting in line", ()
|
||||
expect(afterIssue.executionRunId).toBeNull();
|
||||
expect(busyRun).toBeTruthy();
|
||||
}, 120_000);
|
||||
|
||||
/** Hold the task row in another transaction until `release` is called. */
|
||||
async function holdTaskRow(issueId: string) {
|
||||
let release!: () => void;
|
||||
const released = new Promise<void>((resolve) => { release = resolve; });
|
||||
let locked!: () => void;
|
||||
const lockTaken = new Promise<void>((resolve) => { locked = resolve; });
|
||||
const holder = db.transaction(async (tx) => {
|
||||
await tx.select({ id: issues.id }).from(issues).where(eq(issues.id, issueId)).for("update");
|
||||
locked();
|
||||
await released;
|
||||
});
|
||||
await lockTaken;
|
||||
return { release, holder };
|
||||
}
|
||||
|
||||
async function queueBehindBusyTask() {
|
||||
const { companyId, agentId } = await seed();
|
||||
const [busyIssue] = await db.insert(issues).values({ companyId, title: "Busy", status: "todo", assigneeAgentId: agentId }).returning();
|
||||
gates.set(busyIssue.id, null);
|
||||
await heartbeat.wakeup(agentId, { source: "assignment", triggerDetail: "system", reason: "issue_assigned",
|
||||
requestedByActorType: "system", payload: { issueId: busyIssue.id } });
|
||||
await vi.waitFor(() => expect(executedIssueIds()).toContain(busyIssue.id), { timeout: 10_000 });
|
||||
const waiting = await assignAndWake(companyId, agentId, "Waiting task");
|
||||
const queued = await heartbeat.getRun(waiting.run.id);
|
||||
expect(queued?.status).toBe("queued");
|
||||
expect(queued?.runtimeMode).toBe("legacy");
|
||||
return waiting;
|
||||
}
|
||||
|
||||
// The board route attributes Stop to the user, which first supersedes the
|
||||
// task's handoffs under the same task-row lock.
|
||||
const stopCallers = [
|
||||
{ caller: "control plane", reason: "Cancelled by control plane", options: undefined },
|
||||
{ caller: "board", reason: "Cancelled by a board operator",
|
||||
options: { resultJson: { cancelledByActorType: "user", cancelledByUserId: "responsible-user" } } },
|
||||
] as const;
|
||||
|
||||
it.each(stopCallers)("stops a waiting legacy run for the $caller when another writer briefly holds its task row past the lock timeout", async ({ reason, options }) => {
|
||||
const waiting = await queueBehindBusyTask();
|
||||
const lock = await holdTaskRow(waiting.issue.id);
|
||||
// Longer than one 1s lock_timeout, shorter than the bounded attempts.
|
||||
const releaseTimer = setTimeout(lock.release, 1_500);
|
||||
try {
|
||||
const stopped = await heartbeat.cancelRun(waiting.run.id, reason, options);
|
||||
expect(stopped?.status).toBe("cancelled");
|
||||
} finally {
|
||||
clearTimeout(releaseTimer);
|
||||
lock.release();
|
||||
await lock.holder;
|
||||
}
|
||||
expect((await heartbeat.getRun(waiting.run.id))?.status).toBe("cancelled");
|
||||
}, 60_000);
|
||||
|
||||
it.each(stopCallers)("answers the $caller a retryable conflict when the task row stays locked through every attempt", async ({ reason, options }) => {
|
||||
const waiting = await queueBehindBusyTask();
|
||||
const lock = await holdTaskRow(waiting.issue.id);
|
||||
try {
|
||||
const startedAt = Date.now();
|
||||
const error = await heartbeat.cancelRun(waiting.run.id, reason, options).then(
|
||||
() => null, (caught: unknown) => caught);
|
||||
expect(error).toBeInstanceOf(HttpError);
|
||||
expect((error as HttpError).status).toBe(409);
|
||||
expect((error as HttpError).details).toMatchObject({ code: "run_stop_busy", runId: waiting.run.id });
|
||||
// Bounded: three 1s waits plus short backoff, never an open-ended queue.
|
||||
expect(Date.now() - startedAt).toBeLessThan(10_000);
|
||||
} finally {
|
||||
lock.release();
|
||||
await lock.holder;
|
||||
}
|
||||
// Nothing was committed, so the same Stop succeeds once the writer is done.
|
||||
expect((await heartbeat.getRun(waiting.run.id))?.status).toBe("queued");
|
||||
const stopped = await heartbeat.cancelRun(waiting.run.id, reason, options);
|
||||
expect(stopped?.status).toBe("cancelled");
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -72,6 +72,7 @@ import {
|
||||
waitForAdapterStop,
|
||||
} from "../adapter-execution-control.js";
|
||||
import { AdapterStopTimeoutError } from "../adapter-stop-timeout.js";
|
||||
import { isTransientLockContention, postgresErrorCode } from "../transient-lock-contention.js";
|
||||
import { pendingBudgetRunCancellationPredicate, pendingBudgetRunIdsForScopeQuery } from "./budget-cancellation.js";
|
||||
import { notifyBudgetStopsSettled } from "../budget-resume-continuation.js";
|
||||
import { withBudgetSpan } from "../budget-tracing.js";
|
||||
@@ -1249,10 +1250,19 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
let caseHandoffRetryClaimed = false;
|
||||
if (handoffIssueId && supersedesCaseHandoff && options.resultJson?.caseHandoffYieldStop !== true) {
|
||||
const publications: ActivityPublication[] = [];
|
||||
let taskRowLocked = false;
|
||||
const applySupersession = () => db.transaction(async tx => {
|
||||
taskRowLocked = false;
|
||||
// As in legacy terminalization, wait at most 1s for the task row so a
|
||||
// board Stop never queues behind a slow task writer. Later locks keep
|
||||
// the transaction's previous timeout.
|
||||
const [settings] = await tx.execute(sql`select current_setting('lock_timeout') as lock_timeout`);
|
||||
await tx.execute(sql`select set_config('lock_timeout', '1000', true)`);
|
||||
const [issue] = await tx.select({ id: issues.id, executionRunId: issues.executionRunId, checkoutRunId: issues.checkoutRunId,
|
||||
assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId }).from(issues)
|
||||
.where(and(eq(issues.companyId, run!.companyId), eq(issues.id, handoffIssueId))).for("update");
|
||||
taskRowLocked = true;
|
||||
await tx.execute(sql`select set_config('lock_timeout', ${String(settings.lock_timeout)}, true)`);
|
||||
const [current] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, runId),
|
||||
eq(heartbeatRuns.companyId, run!.companyId))).for("update");
|
||||
if (!current || (options.expectedStatus && current.status !== options.expectedStatus)) return false;
|
||||
@@ -1313,9 +1323,18 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
supersessionApplied = await applySupersession();
|
||||
break;
|
||||
} catch (error) {
|
||||
const value = parseObject(error);
|
||||
const deadlock = value.code === "40P01" || parseObject(value.cause).code === "40P01";
|
||||
if (!deadlock || attempt >= 2) rethrowNativeCancellationLockConflict(error);
|
||||
const code = postgresErrorCode(error);
|
||||
const taskRowBusy = code === "55P03" && !taskRowLocked;
|
||||
if ((code !== "40P01" && !taskRowBusy) || attempt >= 2) {
|
||||
// The task row stayed locked through every attempt. Nothing was
|
||||
// committed, so the same Stop can be sent again.
|
||||
if (taskRowBusy) {
|
||||
throw conflict("The task is busy, so Stop could not finish. Try Stop again.", {
|
||||
code: "run_stop_busy", runId,
|
||||
});
|
||||
}
|
||||
rethrowNativeCancellationLockConflict(error);
|
||||
}
|
||||
// PostgreSQL aborted the entire transaction. Discard its local
|
||||
// receipts before re-reading current authority under fresh locks.
|
||||
// Provider Stop and publication happen only after this boundary.
|
||||
@@ -1646,6 +1665,13 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
);
|
||||
}
|
||||
}
|
||||
// Another writer held the task row through every bounded attempt.
|
||||
// The terminal write rolled back, so the same Stop can be sent again.
|
||||
if (isTransientLockContention(error)) {
|
||||
throw conflict("The task is busy, so Stop could not finish. Try Stop again.", {
|
||||
code: "run_stop_busy", runId: run.id,
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
if (processCancellationSettlement) {
|
||||
|
||||
@@ -59,6 +59,7 @@ import {
|
||||
isConversationAdapter,
|
||||
} from "../conversation-continuation.js";
|
||||
import { terminalizeLegacyExecution } from "../legacy-execution-recovery.js";
|
||||
import { retryTransientLockContention } from "../transient-lock-contention.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
|
||||
import { buildHeartbeatRunStatusLiveEventPayload } from "../heartbeat-run-status-payload.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
@@ -289,6 +290,9 @@ const PRE_ADAPTER_SETUP_FAILURE_CODES = new Set<string>([
|
||||
...NON_RETRYABLE_PREFLIGHT_FAILURE_CODES,
|
||||
]);
|
||||
|
||||
/** Each attempt waits at most the 1s task-row lock_timeout, so a Stop stays prompt. */
|
||||
const LEGACY_TERMINALIZATION_LOCK_ATTEMPTS = 3;
|
||||
|
||||
const MAX_TURN_CONTINUATION_DEFAULT_MAX_ATTEMPTS = 2;
|
||||
|
||||
const MAX_TURN_CONTINUATION_MAX_ATTEMPTS_CAP = 10;
|
||||
@@ -1279,7 +1283,8 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl
|
||||
},
|
||||
};
|
||||
}
|
||||
const updated = await writeRunStatus(status, previousStatus, async writer => {
|
||||
const legacyTerminalization = previousStatus?.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(status);
|
||||
const write = () => writeRunStatus(status, previousStatus, async writer => {
|
||||
if (options?.setupFailure && previousStatus?.runtimeMode === "native" && previousStatus.nativeIssueId) {
|
||||
await writer.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations)
|
||||
.where(and(eq(nativeRunFinalizations.runId, runId),
|
||||
@@ -1292,7 +1297,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl
|
||||
eq(nativeRunFinalizations.companyId, previousStatus.companyId)))
|
||||
.for("update");
|
||||
}
|
||||
return previousStatus && previousStatus.runtimeMode === "legacy" && isHeartbeatRunTerminalStatus(status)
|
||||
return previousStatus && legacyTerminalization
|
||||
? terminalizeLegacyExecution({
|
||||
db: writer,
|
||||
run: previousStatus,
|
||||
@@ -1347,6 +1352,12 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl
|
||||
throw error;
|
||||
});
|
||||
});
|
||||
// Legacy terminalization takes the task row under a short lock_timeout so
|
||||
// it never queues behind a slow task writer. Losing that race rolls back
|
||||
// the whole transaction, so a bounded fresh attempt is safe.
|
||||
const updated = legacyTerminalization
|
||||
? await retryTransientLockContention(write, LEGACY_TERMINALIZATION_LOCK_ATTEMPTS)
|
||||
: await write();
|
||||
|
||||
if (updated) {
|
||||
publishLiveEvent({
|
||||
|
||||
Reference in new issue
Block a user