diff --git a/server/src/services/heartbeat/queue.test.ts b/server/src/services/heartbeat/queue.test.ts index 65a6a15244f..fd9afa56d83 100644 --- a/server/src/services/heartbeat/queue.test.ts +++ b/server/src/services/heartbeat/queue.test.ts @@ -1202,6 +1202,52 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { expect(deps.executeRun).toHaveBeenCalledExactlyOnceWith(run.id); }); + it("starts each claimed run before claiming the next, within the agent's concurrency cap", async () => { + // A claim commits a 60 s legacy controller lease that only the run's own + // execution renews. A batch whose later claims are slow must not hold the + // earlier claimed runs back from executing until their leases lapse. + const [company] = await db.insert(companies).values({ name: "Claim batch", issuePrefix: "CB" }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Busy agent", role: "engineer", status: "idle", + adapterType: "process", adapterConfig: {}, runtimeConfig: { heartbeat: { maxConcurrentRuns: 2 } } }).returning(); + const queued = []; + for (let index = 0; index < 3; index++) { + const [issue] = await db.insert(issues).values({ companyId: company.id, title: `Batch task ${index}`, status: "todo", + assigneeAgentId: agent.id }).returning(); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: company.id, agentId: agent.id, source: "assignment", + status: "claimed", claimedAt: new Date() }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "queued", + invocationSource: "assignment", wakeupRequestId: wake.id, responsibleUserId: "queue-test-user", contextSnapshot: { issueId: issue.id }, + createdAt: new Date(Date.now() - (3 - index) * 1_000) }).returning(); + await db.update(issues).set({ executionRunId: run.id, executionAgentNameKey: agent.name, executionLockedAt: new Date() }) + .where(eq(issues.id, issue.id)); + queued.push(run); + } + const deps = callbacks(db); + deps.getAgent.mockResolvedValue(agent); + let releaseSecondClaim!: () => void; + const secondClaimMayContinue = new Promise(resolve => { releaseSecondClaim = resolve; }); + let budgetChecks = 0; + deps.budgets.getInvocationBlock.mockImplementation(async () => { + budgetChecks += 1; + if (budgetChecks === 2) await secondClaimMayContinue; + return null; + }); + const admission = vi.fn(() => false); + deps.isRunAdmissionClosed = admission; + const pass = createHeartbeatQueue(db, deps).startNextQueuedRunForAgent(agent.id); + try { + await vi.waitFor(() => expect(budgetChecks).toBe(2)); + // The first run is executing while the second claim is still in progress. + expect(deps.executeRun).toHaveBeenCalledExactlyOnceWith(queued[0]!.id); + } finally { releaseSecondClaim(); } + const claimed = await pass; + expect(claimed.map(run => run.id)).toEqual([queued[0]!.id, queued[1]!.id]); + expect(deps.executeRun.mock.calls.map(([runId]) => runId)).toEqual([queued[0]!.id, queued[1]!.id]); + // The run-start admission gate is checked before every claim, and the cap holds. + expect(admission.mock.calls.length).toBeGreaterThanOrEqual(3); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queued[2]!.id)))[0].status).toBe("queued"); + }); + it("leaves a queued run for the next server when shutdown closes admission during the queue scan", async () => { const { run, agent, wake } = await fixture("queued"); await db.update(heartbeatRuns).set({ responsibleUserId: "queue-test-user" }).where(eq(heartbeatRuns.id, run.id)); diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 40d2af26f65..e1ac0519ff0 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -2589,17 +2589,21 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende noteDeclinedAgedClaim(queuedRun, err.message); continue; } - if (claimed) { - claimedRuns.push(claimed); - dispatcher.claimed(queuedRun); - declinedAgedClaimAt.delete(queuedRun.id); - } else { + if (!claimed) { noteDeclinedAgedClaim(queuedRun, "claim_declined"); + continue; } + claimedRuns.push(claimed); + dispatcher.claimed(queuedRun); + declinedAgedClaimAt.delete(queuedRun.id); + // Start each run as soon as it is claimed. The claim committed a legacy + // controller lease that only the run's own execution renews; waiting + // for the rest of a slow batch to be claimed let that lease lapse. + startClaimedRun(claimed); } - if (claimedRuns.length === 0) return []; + return claimedRuns; - for (const claimedRun of claimedRuns) { + function startClaimedRun(claimedRun: typeof heartbeatRuns.$inferSelect) { const execution = executeRun(claimedRun.id).catch((err) => { logger.error( { err, runId: claimedRun.id }, @@ -2617,7 +2621,6 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende activeRunExecutionPromises.delete(execution); }); } - return claimedRuns; }).finally(async () => { // Dispatch promoted inputs after the agent start lock is released. try { diff --git a/server/src/services/legacy-controller-lease.test.ts b/server/src/services/legacy-controller-lease.test.ts index 0ff1028f38a..81971ac2107 100644 --- a/server/src/services/legacy-controller-lease.test.ts +++ b/server/src/services/legacy-controller-lease.test.ts @@ -189,6 +189,42 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(await renewLegacyControllerLease(db, run)).toBe(false); expect(await revokeExpiredLegacyController(db, run)).toBe(false); }); + it("renews at once when the watch starts on a lease older than one renewal interval", async () => { + // The watch can start long after the claim (a slow claim batch). Waiting for + // the first 10 s tick let a still-live lease reach its deadline unrenewed. + const run = await seed(); + const [late] = await db.update(heartbeatRuns).set({ controllerLeaseExpiresAt: sql`clock_timestamp() + interval '1500 milliseconds'` }) + .where(eq(heartbeatRuns.id, run.id)).returning(); + const controller = new AbortController(); + const watch = watchLegacyControllerLease(db, late, controller); + try { + await new Promise(resolve => setTimeout(resolve, 2_000)); + expect(controller.signal.reason?.leaseLossTiming).toBeUndefined(); + expect(controller.signal.aborted).toBe(false); + const [saved] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + expect(saved.controllerLeaseExpiresAt!.getTime()).toBeGreaterThan(Date.now() + LEGACY_CONTROLLER_LEASE_MS - 5_000); + expect(saved.controllerBootId).toBe(legacyControllerBootId); + } finally { watch.stop(); } + }); + it.each(["revoked by recovery", "reaped", "lapsed"] as const)( + "a watch that starts late on a lease that was %s still loses it and never revives it", async outcome => { + const run = await seed(); + await expire(run.id); + if (outcome === "revoked by recovery") expect(await revokeExpiredLegacyController(db, run)).toBe(true); + if (outcome === "reaped") await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); + const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + const controller = new AbortController(); + // The stale claim-time row still names this boot with a lease that has not expired locally. + const watch = watchLegacyControllerLease(db, { ...run, controllerLeaseExpiresAt: new Date(Date.now() + 1_000) }, controller); + try { + await vi.waitFor(() => expect(controller.signal.aborted).toBe(true), { timeout: 2_000 }); + expect(controller.signal.reason).toBeInstanceOf(LegacyControllerLeaseLostError); + expect(controller.signal.reason.leaseLossCause).toBe("renewal_rejected"); + const [after] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + expect(after).toMatchObject({ status: before.status, controllerBootId: before.controllerBootId, + controllerLeaseExpiresAt: before.controllerLeaseExpiresAt }); + } finally { watch.stop(); } + }); it("rejects dispatch at the lease deadline even if the database query never settles", async () => { const run = await seed(); const hungDb = fakeRenewalDb(() => new Promise(() => {})); @@ -619,7 +655,9 @@ describe("legacy controller lease loss attribution", () => { await new Promise((resolve) => setTimeout(resolve, 5)); expect(controller.signal.reason.leaseLossCause).toBe("renewal_deadline"); expect(controller.signal.reason.leaseLossTiming.deadlineLateMs).toBeGreaterThanOrEqual(60); - expect(controller.signal.reason.leaseLossTiming.renewalPendingMs).toBeNull(); + // A watch that starts this close to the deadline renews at once; that renewal never answered. + expect(controller.signal.reason.leaseLossTiming.renewalPendingMs).toBeGreaterThanOrEqual(60); + expect(controller.signal.reason.leaseLossTiming.renewalCount).toBe(0); } finally { watch.stop(); } }); }); diff --git a/server/src/services/legacy-controller-lease.ts b/server/src/services/legacy-controller-lease.ts index 407c579a792..109ee8f54b9 100644 --- a/server/src/services/legacy-controller-lease.ts +++ b/server/src/services/legacy-controller-lease.ts @@ -390,7 +390,8 @@ export function watchLegacyControllerLease( deadline = setTimeout(deadlinePassed, delayMs); deadline.unref(); }; - armDeadline(Math.max(0, (run.controllerLeaseExpiresAt?.getTime() ?? 0) - Date.now())); + const initialRemainingMs = (run.controllerLeaseExpiresAt?.getTime() ?? 0) - Date.now(); + armDeadline(Math.max(0, initialRemainingMs)); const assertOwned = async (stage?: "dispatching") => { if (stopped) return; controller.signal.throwIfAborted(); @@ -447,11 +448,19 @@ export function watchLegacyControllerLease( armDeadline(Math.max(0, LEGACY_CONTROLLER_LEASE_MS - (Date.now() - renewedFrom))); } }; - const timer = setInterval(() => { + const tick = () => { if (pending || stopped) return; pending = true; void assertOwned().catch(() => lost("renewal_error")).finally(() => { pending = false; }); - }, LEGACY_CONTROLLER_RENEW_MS); + }; + const timer = setInterval(tick, LEGACY_CONTROLLER_RENEW_MS); timer.unref(); + // The watch can start well after the claim that wrote the lease (a slow + // batch of claims, a loaded database). A lease older than one renewal + // interval is renewed now rather than on the next tick. The renewal is the + // same conditional statement as every other one: it only extends a lease + // this boot still holds on a running row, so a lease that lapsed and was + // revoked, reaped or taken over is still lost. + if (initialRemainingMs < LEGACY_CONTROLLER_LEASE_MS - LEGACY_CONTROLLER_RENEW_MS) tick(); return { assertOwned, stop() { stopped = true; clearInterval(timer); clearTimeout(deadline); } }; }