diff --git a/server/src/services/heartbeat/retries.test.ts b/server/src/services/heartbeat/retries.test.ts index ab5cb054f6c..dbd8186f3c9 100644 --- a/server/src/services/heartbeat/retries.test.ts +++ b/server/src/services/heartbeat/retries.test.ts @@ -475,3 +475,19 @@ describePostgres("heartbeat retry module database wiring", () => { expect(await retries.retryScheduledRetryNow({ issueId: selected.issue.id, now })).toMatchObject({ outcome: "already_promoted" }); }); }); + +describe("interrupted preparation retry schedule", () => { + it("retries a preparation a restart interrupted at once, and backs off only if it keeps being interrupted", () => { + const now = new Date("2026-11-09T11:09:41.846Z"); + const first = extracted.computeInterruptedPreparationRetrySchedule(1, now, () => 0.5); + expect(first).toMatchObject({ attempt: 1, delayMs: 0, dueAt: now }); + const second = extracted.computeInterruptedPreparationRetrySchedule(2, now, () => 0.5); + expect(second?.delayMs).toBeGreaterThan(0); + expect(second).toMatchObject({ attempt: 2, baseDelayMs: extracted.BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS[0] }); + // Later attempts keep the longest transient delay. + expect(extracted.computeInterruptedPreparationRetrySchedule(5, now, () => 0.5)) + .toMatchObject({ attempt: 5, baseDelayMs: extracted.BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.at(-1) }); + expect(extracted.computeInterruptedPreparationRetrySchedule(0, now)).toBeNull(); + }); +}); + diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 4308d403e98..f8738e9d16c 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -405,6 +405,28 @@ export function computeBoundedTransientHeartbeatRetrySchedule( }; } +/** + * The interrupted-preparation lane: a preparation that a restart interrupted + * spent no provider work and hit no transient fault, so its first retry starts + * at once. If it is interrupted again it backs off like a transient failure + * (the longest delay from then on), so a preparation that itself keeps taking + * the server down is slowed and still capped by the lane's attempt limit. + */ +export function computeInterruptedPreparationRetrySchedule( + attempt: number, + now = new Date(), + random: () => number = Math.random, +) { + if (!Number.isInteger(attempt) || attempt <= 0) return null; + if (attempt === 1) { + return { attempt, baseDelayMs: 0, delayMs: 0, dueAt: new Date(now.getTime()), + maxAttempts: BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS }; + } + const schedule = computeBoundedTransientHeartbeatRetrySchedule( + Math.min(attempt - 1, BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS), now, random); + return schedule ? { ...schedule, attempt } : null; +} + export { normalizeAgentNameKey }; function readNonEmptyString(value: unknown): string | null { @@ -515,12 +537,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen : null : nextAttempt <= maxAttempts ? interruptedPreparationLane - ? (() => { - // Later lane attempts keep the longest transient delay. - const schedule = computeBoundedTransientHeartbeatRetrySchedule( - Math.min(nextAttempt, BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS), now, opts?.random); - return schedule ? { ...schedule, attempt: nextAttempt } : null; - })() + ? computeInterruptedPreparationRetrySchedule(nextAttempt, now, opts?.random) : computeBoundedTransientHeartbeatRetrySchedule( nextAttempt, now,