From 834588df8feb1e72cc0211612f6e2d7e54c21332 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 16 Jul 2026 15:09:27 +0000 Subject: [PATCH] fix heartbeat timer duplicate run race --- ...heartbeat-stale-queue-invalidation.test.ts | 33 +++++++++++++++++++ server/src/services/heartbeat.ts | 26 +++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts index 35f33c9f460..afa06b06c62 100644 --- a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts +++ b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts @@ -367,6 +367,39 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { expect(agent?.lastHeartbeatAt?.getTime()).toBeGreaterThan(now.getTime() - 120_000); }); + it("atomically claims a due timer interval across overlapping scheduler ticks", async () => { + const { agentId } = await seedCompanyAndAgent({ + heartbeatConfig: { + enabled: true, + intervalSec: 60, + }, + }); + const now = new Date(); + await db + .update(agents) + .set({ lastHeartbeatAt: new Date(now.getTime() - 120_000) }) + .where(eq(agents.id, agentId)); + + const results = await Promise.all([ + heartbeat.tickTimers(now), + heartbeat.tickTimers(now), + ]); + + expect(results.reduce((total, result) => total + result.enqueued, 0)).toBe(1); + + const runs = await db + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.agentId, agentId)); + const [agent] = await db + .select({ lastHeartbeatAt: agents.lastHeartbeatAt }) + .from(agents) + .where(eq(agents.id, agentId)); + + expect(runs).toHaveLength(1); + expect(agent?.lastHeartbeatAt?.getTime()).toBeGreaterThanOrEqual(now.getTime()); + }); + it("allows generic timer wakes when the agent has assigned todo work", async () => { const { companyId, agentId } = await seedCompanyAndAgent({ heartbeatConfig: { diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 47ee92625a0..445b5764e5c 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -10551,6 +10551,31 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .where(eq(agents.id, agentId)); } + async function claimDueTimerHeartbeat( + agent: typeof agents.$inferSelect, + now: Date, + intervalSec: number, + ) { + const dueBefore = new Date(now.getTime() - intervalSec * 1000); + const claimed = await db + .update(agents) + .set({ + lastHeartbeatAt: now, + updatedAt: now, + }) + .where(and( + eq(agents.id, agent.id), + eq(agents.companyId, agent.companyId), + or( + lte(agents.lastHeartbeatAt, dueBefore), + and(isNull(agents.lastHeartbeatAt), lte(agents.createdAt, dueBefore)), + ), + )) + .returning({ id: agents.id }) + .then((rows) => rows[0] ?? null); + return Boolean(claimed); + } + function parseMaxTurnContinuationPolicy(agent: typeof agents.$inferSelect): MaxTurnContinuationPolicy { const runtimeConfig = parseObject(agent.runtimeConfig); const heartbeat = parseObject(runtimeConfig.heartbeat); @@ -16972,6 +16997,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const baseline = new Date(agent.lastHeartbeatAt ?? agent.createdAt).getTime(); const elapsedMs = now.getTime() - baseline; if (elapsedMs < policy.intervalSec * 1000) continue; + if (!(await claimDueTimerHeartbeat(agent, now, policy.intervalSec))) continue; const run = await enqueueWakeup(agent.id, { source: "timer",