fix heartbeat timer duplicate run race

This commit is contained in:
Dotta committed 2026-07-16 15:09:27 +00:00
1 parent c65ab09d9f
commit 834588df8f
2 files changed
+59

No files matched your search

@@ -367,6 +367,39 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
expect(agent?.lastHeartbeatAt?.getTime()).toBeGreaterThan(now.getTime() - 120_000);
});
it("atomically claims a due timer interval across overlapping scheduler ticks", async () => {
const { agentId } = await seedCompanyAndAgent({
heartbeatConfig: {
enabled: true,
intervalSec: 60,
},
});
const now = new Date();
await db
.update(agents)
.set({ lastHeartbeatAt: new Date(now.getTime() - 120_000) })
.where(eq(agents.id, agentId));
const results = await Promise.all([
heartbeat.tickTimers(now),
heartbeat.tickTimers(now),
]);
expect(results.reduce((total, result) => total + result.enqueued, 0)).toBe(1);
const runs = await db
.select({ id: heartbeatRuns.id })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.agentId, agentId));
const [agent] = await db
.select({ lastHeartbeatAt: agents.lastHeartbeatAt })
.from(agents)
.where(eq(agents.id, agentId));
expect(runs).toHaveLength(1);
expect(agent?.lastHeartbeatAt?.getTime()).toBeGreaterThanOrEqual(now.getTime());
});
it("allows generic timer wakes when the agent has assigned todo work", async () => {
const { companyId, agentId } = await seedCompanyAndAgent({
heartbeatConfig: {
+26
View File
@@ -10551,6 +10551,31 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
.where(eq(agents.id, agentId));
}
async function claimDueTimerHeartbeat(
agent: typeof agents.$inferSelect,
now: Date,
intervalSec: number,
) {
const dueBefore = new Date(now.getTime() - intervalSec * 1000);
const claimed = await db
.update(agents)
.set({
lastHeartbeatAt: now,
updatedAt: now,
})
.where(and(
eq(agents.id, agent.id),
eq(agents.companyId, agent.companyId),
or(
lte(agents.lastHeartbeatAt, dueBefore),
and(isNull(agents.lastHeartbeatAt), lte(agents.createdAt, dueBefore)),
),
))
.returning({ id: agents.id })
.then((rows) => rows[0] ?? null);
return Boolean(claimed);
}
function parseMaxTurnContinuationPolicy(agent: typeof agents.$inferSelect): MaxTurnContinuationPolicy {
const runtimeConfig = parseObject(agent.runtimeConfig);
const heartbeat = parseObject(runtimeConfig.heartbeat);
@@ -16972,6 +16997,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
const baseline = new Date(agent.lastHeartbeatAt ?? agent.createdAt).getTime();
const elapsedMs = now.getTime() - baseline;
if (elapsedMs < policy.intervalSec * 1000) continue;
if (!(await claimDueTimerHeartbeat(agent, now, policy.intervalSec))) continue;
const run = await enqueueWakeup(agent.id, {
source: "timer",