mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-09 07:10:34 +01:00
fix heartbeat timer duplicate run race
This commit is contained in:
1 parent
c65ab09d9f
commit
834588df8f
2 files changed
+59
No files matched your search
@@ -367,6 +367,39 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
expect(agent?.lastHeartbeatAt?.getTime()).toBeGreaterThan(now.getTime() - 120_000);
|
||||
});
|
||||
|
||||
it("atomically claims a due timer interval across overlapping scheduler ticks", async () => {
|
||||
const { agentId } = await seedCompanyAndAgent({
|
||||
heartbeatConfig: {
|
||||
enabled: true,
|
||||
intervalSec: 60,
|
||||
},
|
||||
});
|
||||
const now = new Date();
|
||||
await db
|
||||
.update(agents)
|
||||
.set({ lastHeartbeatAt: new Date(now.getTime() - 120_000) })
|
||||
.where(eq(agents.id, agentId));
|
||||
|
||||
const results = await Promise.all([
|
||||
heartbeat.tickTimers(now),
|
||||
heartbeat.tickTimers(now),
|
||||
]);
|
||||
|
||||
expect(results.reduce((total, result) => total + result.enqueued, 0)).toBe(1);
|
||||
|
||||
const runs = await db
|
||||
.select({ id: heartbeatRuns.id })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.agentId, agentId));
|
||||
const [agent] = await db
|
||||
.select({ lastHeartbeatAt: agents.lastHeartbeatAt })
|
||||
.from(agents)
|
||||
.where(eq(agents.id, agentId));
|
||||
|
||||
expect(runs).toHaveLength(1);
|
||||
expect(agent?.lastHeartbeatAt?.getTime()).toBeGreaterThanOrEqual(now.getTime());
|
||||
});
|
||||
|
||||
it("allows generic timer wakes when the agent has assigned todo work", async () => {
|
||||
const { companyId, agentId } = await seedCompanyAndAgent({
|
||||
heartbeatConfig: {
|
||||
|
||||
@@ -10551,6 +10551,31 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
.where(eq(agents.id, agentId));
|
||||
}
|
||||
|
||||
async function claimDueTimerHeartbeat(
|
||||
agent: typeof agents.$inferSelect,
|
||||
now: Date,
|
||||
intervalSec: number,
|
||||
) {
|
||||
const dueBefore = new Date(now.getTime() - intervalSec * 1000);
|
||||
const claimed = await db
|
||||
.update(agents)
|
||||
.set({
|
||||
lastHeartbeatAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(and(
|
||||
eq(agents.id, agent.id),
|
||||
eq(agents.companyId, agent.companyId),
|
||||
or(
|
||||
lte(agents.lastHeartbeatAt, dueBefore),
|
||||
and(isNull(agents.lastHeartbeatAt), lte(agents.createdAt, dueBefore)),
|
||||
),
|
||||
))
|
||||
.returning({ id: agents.id })
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return Boolean(claimed);
|
||||
}
|
||||
|
||||
function parseMaxTurnContinuationPolicy(agent: typeof agents.$inferSelect): MaxTurnContinuationPolicy {
|
||||
const runtimeConfig = parseObject(agent.runtimeConfig);
|
||||
const heartbeat = parseObject(runtimeConfig.heartbeat);
|
||||
@@ -16972,6 +16997,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
|
||||
const baseline = new Date(agent.lastHeartbeatAt ?? agent.createdAt).getTime();
|
||||
const elapsedMs = now.getTime() - baseline;
|
||||
if (elapsedMs < policy.intervalSec * 1000) continue;
|
||||
if (!(await claimDueTimerHeartbeat(agent, now, policy.intervalSec))) continue;
|
||||
|
||||
const run = await enqueueWakeup(agent.id, {
|
||||
source: "timer",
|
||||
|
||||
Reference in new issue
Block a user