From 3d48c4bed219bb861980f565d2d6fc42d4329f4f Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 13 Nov 2026 12:48:26 -0600 Subject: [PATCH] fix(heartbeat): let successors start after their predecessor and stop chaining passes on refused runs Back off a refused claim only when the run has waited past the aging limit, end a backoff when the run's task changes or a run of that task ends, and count a successor's slot without the predecessor it replaces. A pass that reaches its claim budget hands the runs it tried to one follow-up pass, so refused person runs cannot hold back runnable work or chain passes. A trigger after a pass's deadline also ends that pass, measured on a monotonic clock from when it started. Co-Authored-By: Claude Opus 5.5 --- .../queued-run-dispatch-stacking.test.ts | 52 +++++++++++++ server/src/services/agent-start-lock.test.ts | 15 ++++ server/src/services/agent-start-lock.ts | 19 +++-- server/src/services/heartbeat.ts | 1 + server/src/services/heartbeat/queue.ts | 76 ++++++++++++++----- .../services/heartbeat/run-slot-admission.ts | 27 ++++--- .../native-runtime/native-review-dispatch.ts | 2 +- 7 files changed, 155 insertions(+), 37 deletions(-) diff --git a/server/src/__tests__/queued-run-dispatch-stacking.test.ts b/server/src/__tests__/queued-run-dispatch-stacking.test.ts index ca78f15241f..3e9c68488af 100644 --- a/server/src/__tests__/queued-run-dispatch-stacking.test.ts +++ b/server/src/__tests__/queued-run-dispatch-stacking.test.ts @@ -234,4 +234,56 @@ suite("queued-run dispatch under a trigger burst", () => { await heartbeat.drainActiveRunExecutions(); } }, 60_000); + + async function passesSettle() { + let last = -1; + await vi.waitFor(() => { + const settled = passes.started === last && !passes.active; + last = passes.started; + expect(settled).toBe(true); + }, { timeout: 30_000, interval: 1_000 }); + return passes.started; + } + + it("stops chaining passes once every refused run was tried", async () => { + passes.started = 0; passes.active = 0; passes.peak = 0; + const agentId = await seedAgent(); + const otherTask = await task(agentId); + for (let index = 0; index < 60; index += 1) { + await queuedContinuation(agentId, { minutesAgo: 180 - index, wakeReason: "issue_continuation_needed", sourceIssueId: otherTask.id }); + } + const heartbeat = heartbeatService(db); + try { + await heartbeat.resumeQueuedRunsForAgent(companyId, agentId); + expect(await passesSettle()).toBe(3); + expect(passes.peak).toBe(1); + } finally { + await heartbeat.drainActiveRunExecutions(); + } + }, 120_000); + + it("starts runnable work queued behind more refused person runs than one pass tries", async () => { + held.reset(); + passes.started = 0; passes.active = 0; passes.peak = 0; + const agentId = await seedAgent(); + const otherTask = await task(agentId); + for (let index = 0; index < 30; index += 1) { + await queuedContinuation(agentId, { minutesAgo: 90 - index, wakeReason: "task_retry", sourceIssueId: otherTask.id }); + } + const runnable: string[] = []; + for (let index = 0; index < 3; index += 1) { + runnable.push((await queuedContinuation(agentId, { minutesAgo: 30 - index, wakeReason: "issue_assigned" })).id); + } + const heartbeat = heartbeatService(db); + try { + await heartbeat.resumeQueuedRunsForAgent(companyId, agentId); + expect(await passesSettle()).toBeLessThanOrEqual(3); + const started = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .where(sql`${heartbeatRuns.id} in (${sql.join(runnable.map(id => sql`${id}::uuid`), sql`, `)}) and ${heartbeatRuns.startedAt} is not null`); + expect(started).toHaveLength(runnable.length); + } finally { + held.release(); + await heartbeat.drainActiveRunExecutions(); + } + }, 120_000); }); diff --git a/server/src/services/agent-start-lock.test.ts b/server/src/services/agent-start-lock.test.ts index dee3471a2d5..578967249f1 100644 --- a/server/src/services/agent-start-lock.test.ts +++ b/server/src/services/agent-start-lock.test.ts @@ -95,6 +95,21 @@ describe("agent dispatch passes", () => { expect(await dispatchForAgent(agentId, spec(false, async () => undefined))).toBe(3); }); + it("lets a trigger stop a hung pass by the pass's start time when its deadline timer has not run", async () => { + vi.useFakeTimers({ toFake: ["performance"] }); + const agentId = randomUUID(); + const { stats, spec } = meter(); + let hungSignal: AbortSignal | null = null; + void dispatchForAgent(agentId, spec(false, (signal) => { + hungSignal = signal; + return new Promise(() => undefined); + })); + vi.advanceTimersByTime(AGENT_DISPATCH_PASS_DEADLINE_MS); + expect(await dispatchForAgent(agentId, spec(false, async () => undefined))).toBe(2); + expect(hungSignal!.aborted).toBe(true); + expect(stats.passes).toBe(2); + }); + it("does not wait on itself when a pass triggers its own agent", async () => { const agentId = randomUUID(); const { stats, spec } = meter(); diff --git a/server/src/services/agent-start-lock.ts b/server/src/services/agent-start-lock.ts index 9b4aa4abcd0..70f5db23284 100644 --- a/server/src/services/agent-start-lock.ts +++ b/server/src/services/agent-start-lock.ts @@ -13,7 +13,7 @@ export type AgentDispatchPass = { type Deferred = { promise: Promise; resolve(value: T): void; reject(err: unknown): void }; type FollowUp = { request: unknown; spec: AgentDispatchPass; done: Deferred }; -type AgentDispatchState = { followUp: FollowUp | null }; +type AgentDispatchState = { followUp: FollowUp | null; startedAtMs: number; expire(): void }; const dispatchStates = new Map(); const dispatchingAgent = new AsyncLocalStorage(); @@ -38,7 +38,11 @@ export function outsideAgentDispatch(fn: () => T): T { } export function dispatchForAgent(agentId: string, spec: AgentDispatchPass): Promise { - const state = dispatchStates.get(agentId); + let state = dispatchStates.get(agentId); + if (state && performance.now() - state.startedAtMs >= AGENT_DISPATCH_PASS_DEADLINE_MS) { + state.expire(); + state = dispatchStates.get(agentId); + } if (!state) return startPass(agentId, spec, spec.request); if (state.followUp) { state.followUp.request = spec.merge(state.followUp.request as R, spec.request); @@ -51,10 +55,10 @@ export function dispatchForAgent(agentId: string, spec: AgentDispatchPass< } function startPass(agentId: string, spec: AgentDispatchPass, request: R): Promise { - const state: AgentDispatchState = { followUp: null }; - dispatchStates.set(agentId, state); const controller = new AbortController(); const expired = deferred(); + const state: AgentDispatchState = { followUp: null, startedAtMs: performance.now(), expire: () => expire() }; + dispatchStates.set(agentId, state); let released = false; const release = () => { if (released) return; @@ -64,13 +68,16 @@ function startPass(agentId: string, spec: AgentDispatchPass, request state.followUp = null; if (next) startPass(agentId, next.spec, next.request).then(next.done.resolve, next.done.reject); }; - const deadline = setTimeout(() => { + const expire = () => { + if (released) return; + clearTimeout(deadline); logger.warn({ agentId, deadlineMs: AGENT_DISPATCH_PASS_DEADLINE_MS }, "queued-run dispatch pass passed its deadline; it stops claiming and the next pass may start"); controller.abort(); release(); expired.resolve(); - }, AGENT_DISPATCH_PASS_DEADLINE_MS); + }; + const deadline = setTimeout(expire, AGENT_DISPATCH_PASS_DEADLINE_MS); deadline.unref?.(); const pass = dispatchingAgent.run(agentId, () => spec.run(request, controller.signal)).then( async ({ value, settle }) => { diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 23bfa9766c1..6682de39c4d 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -4388,6 +4388,7 @@ export function heartbeatService( logger.warn({ err, runId: run.id }, "failed to deliver settled tool reviews after execution cleanup"); }); } + heartbeatQueue.releaseClaimBackoffsForTask(latestRun ?? run); await startNextQueuedRunForAgent(run.agentId); if (latestRun && isHeartbeatRunTerminalStatus(latestRun.status)) { await notifyOtherAgentsQueuedOnSettledIssue(db, latestRun).catch(err => { diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 2ea2c60154a..ac5cebb0c5f 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -216,8 +216,9 @@ import { crashPoint } from "@paperclipai/shared/crash-points"; type HeartbeatRun = typeof heartbeatRuns.$inferSelect; type QueuedRunCandidate = Pick; -type QueuedDispatchRequest = { attachSetupRequest: boolean; retryRefused: boolean }; + | "startedAt" | "wakeupRequestId" | "retryOfRunId" | "scheduledRetryAt" | "scheduledRetryAttempt" | "scheduledRetryReason" + | "contextSnapshot">; +type QueuedDispatchRequest = { attachSetupRequest: boolean; retryRefused: boolean; triedRunIds: ReadonlySet }; type HeartbeatRunState = ReturnType; type RunPreparation = ReturnType; @@ -382,6 +383,7 @@ const CONVERSATION_COMMENT_KEY_PREFIX = "conversation-comment:"; const QUEUED_RUN_CLAIM_ATTEMPTS_PER_PASS = 24; const QUEUED_RUN_CLAIM_BACKOFF_BASE_MS = 5_000; const QUEUED_RUN_CLAIM_BACKOFF_MAX_MS = 60_000; +const QUEUED_PERSON_RUN_CLAIM_BACKOFF_MS = 2_000; function dailyLimitConversationRefusal(cap: { reason: string; observed: number; limit: number }) { // No receipt is written, so the person's message stays owed in the outbox // until the limit resets (agent-conversations.ts conversationDeliveryHold). @@ -1593,7 +1595,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende // live-comment discard gate below. await withExternalAdmissionGuard(tx, run.companyId, run.agentId, async () => { await assertNoExternalOverlap(tx, run.companyId, run.agentId); - await assertAgentRunSlotFree(tx, run.companyId, run.agentId); + await assertAgentRunSlotFree(tx, run); }); await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); const [claimedRun] = await tx @@ -1692,7 +1694,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende await withExternalAdmissionGuard(tx, run.companyId, run.agentId, async () => { await assertNoExternalOverlap(tx, run.companyId, run.agentId); - await assertAgentRunSlotFree(tx, run.companyId, run.agentId); + await assertAgentRunSlotFree(tx, run); }); await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); await tx @@ -1815,7 +1817,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende if (!lockedRun || lockedRun.status !== "queued") return null; await withExternalAdmissionGuard(claimTx, run.companyId, run.agentId, async () => { await assertNoExternalOverlap(claimTx, run.companyId, run.agentId); - await assertAgentRunSlotFree(claimTx, run.companyId, run.agentId); + await assertAgentRunSlotFree(claimTx, run); }); await notifyChatRunMilestones(claimTx, { companyId: run.companyId, runId: run.id, issueId: issueId ?? undefined }); const claimedRun = await claimTx.update(heartbeatRuns).set(claimValues).where(and( @@ -2370,16 +2372,36 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende "long-queued heartbeat run was offered a free slot but could not be claimed; it stays queued and the next oldest run takes its turn"); } - const claimBackoffs = new Map(); + const claimBackoffs = new Map(); - function claimBackedOff(runId: string, now: Date) { - return (claimBackoffs.get(runId)?.until ?? 0) > now.getTime(); + function claimBackedOff(runId: string, now: Date, taskStamp: string | null) { + const backoff = claimBackoffs.get(runId); + return Boolean(backoff && backoff.until > now.getTime() && backoff.taskStamp === taskStamp); } - function backOffClaim(run: QueuedRunCandidate) { + function backOffClaim(run: QueuedRunCandidate, personRun: boolean, taskStamp: string | null) { + if (!hasQueuedRunAged(run, new Date())) return; const attempts = (claimBackoffs.get(run.id)?.attempts ?? 0) + 1; - const delayMs = Math.min(QUEUED_RUN_CLAIM_BACKOFF_BASE_MS * 2 ** (attempts - 1), QUEUED_RUN_CLAIM_BACKOFF_MAX_MS); - claimBackoffs.set(run.id, { agentId: run.agentId, attempts, until: Date.now() + delayMs }); + const delayMs = personRun ? QUEUED_PERSON_RUN_CLAIM_BACKOFF_MS + : Math.min(QUEUED_RUN_CLAIM_BACKOFF_BASE_MS * 2 ** (attempts - 1), QUEUED_RUN_CLAIM_BACKOFF_MAX_MS); + claimBackoffs.set(run.id, { agentId: run.agentId, issueId: readNonEmptyString(parseObject(run.contextSnapshot).issueId), + attempts, until: Date.now() + delayMs, taskStamp }); + } + + function releaseClaimBackoffsForTask(run: Pick) { + const issueId = run.issueId ?? readNonEmptyString(parseObject(run.contextSnapshot).issueId); + if (!issueId) return; + for (const [runId, backoff] of claimBackoffs) { + if (backoff.issueId === issueId) claimBackoffs.delete(runId); + } + } + + function queuedRunTaskStamp(run: QueuedRunCandidate, + issueById: Map) { + const issueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); + const issue = issueId ? issueById.get(issueId) : null; + return issue ? `${issue.status}:${issue.updatedAt.getTime()}:${issue.executionRunId ?? ""}` : null; } function forgetSettledClaimBackoffs(agentId: string, queuedRuns: QueuedRunCandidate[]) { @@ -2399,13 +2421,14 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende agent: typeof agents.$inferSelect, queuedRuns: QueuedRunCandidate[], dependencyReadiness: Map, - issueById: Map, + issueById: Map, ) { const now = new Date(); pruneDeclinedAgedClaims(now); + const runById = new Map(queuedRuns.map(run => [run.id, run])); const { facts, dispatchInput } = await queuedRunDispatchFacts(db, { agent, queuedRuns, dependencyReadiness, issueById, now, - shareDeferred: runId => declinedAgedClaimAt.has(runId) || claimBackedOff(runId, now) }); + shareDeferred: runId => declinedAgedClaimAt.has(runId) || claimBackedOff(runId, now, queuedRunTaskStamp(runById.get(runId)!, issueById)) }); const dispatcher = createQueuedRunDispatcher(facts, dispatchInput); const byRun = new Map(facts.map(entry => [entry.run, entry])); return { @@ -2420,12 +2443,17 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende } function startNextQueuedRunForAgent(agentId: string, attachSetupRequest = false, retryRefused = false) { + return requestQueuedDispatch(agentId, { attachSetupRequest, retryRefused, triedRunIds: new Set() }); + } + + function requestQueuedDispatch(agentId: string, request: QueuedDispatchRequest) { return dispatchForAgent>(agentId, { - request: { attachSetupRequest, retryRefused }, + request, idle: [], merge: (current, next) => ({ attachSetupRequest: current.attachSetupRequest || next.attachSetupRequest, retryRefused: current.retryRefused || next.retryRefused, + triedRunIds: new Set([...current.triedRunIds, ...next.triedRunIds]), }), run: (request, signal) => runQueuedDispatchPass(agentId, request, signal), track: (pass) => { @@ -2436,7 +2464,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende }); } - async function runQueuedDispatchPass(agentId: string, { attachSetupRequest, retryRefused }: QueuedDispatchRequest, signal: AbortSignal) { + async function runQueuedDispatchPass(agentId: string, { attachSetupRequest, retryRefused, triedRunIds }: QueuedDispatchRequest, + signal: AbortSignal) { // Cancelled after the pass ends: cancelRunInternal promotes the agent's // next queued run, which asks for another pass of this agent. const rejectedClaims: Array<{ run: typeof heartbeatRuns.$inferSelect; err: HttpError }> = []; @@ -2495,7 +2524,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende return issueId ? [issueId] : []; }))]; const issueRows = queuedIssueIds.length > 0 - ? await db.select({ id: issues.id, status: issues.status, priority: issues.priority }).from(issues) + ? await db.select({ id: issues.id, status: issues.status, priority: issues.priority, updatedAt: issues.updatedAt, + executionRunId: issues.executionRunId }).from(issues) .where(and(eq(issues.companyId, agent.companyId), inArray(issues.id, queuedIssueIds))) : []; const issueById = new Map(issueRows.map((row) => [row.id, row])); @@ -2505,14 +2535,18 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende const claimedRuns: Array = []; const slots = { full: false }; let claimAttempts = 0; + const triedThisPass: string[] = []; for (let candidate = dispatcher.next(); candidate; candidate = dispatcher.next()) { if (claimedRuns.length >= availableSlots || slots.full || isRunAdmissionClosed() || signal.aborted) break; - if (!retryRefused && !dispatcher.personRun(candidate) && claimBackedOff(candidate.id, new Date())) continue; + const taskStamp = queuedRunTaskStamp(candidate, issueById); + if (triedRunIds.has(candidate.id) || (!retryRefused && claimBackedOff(candidate.id, new Date(), taskStamp))) continue; if (claimAttempts >= QUEUED_RUN_CLAIM_ATTEMPTS_PER_PASS) { - void startNextQueuedRunForAgent(agentId); + void requestQueuedDispatch(agentId, { attachSetupRequest: false, retryRefused, + triedRunIds: new Set([...triedRunIds, ...triedThisPass]) }); break; } claimAttempts += 1; + triedThisPass.push(candidate.id); const [queuedRun] = await db.select().from(heartbeatRuns).where(and( eq(heartbeatRuns.id, candidate.id), eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.status, "queued"))); if (!queuedRun) continue; @@ -2530,13 +2564,13 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende "queued heartbeat run claim was rejected; leaving it queued for the next recovery pass", ); noteDeclinedAgedClaim(candidate, err.message); - backOffClaim(candidate); + backOffClaim(candidate, dispatcher.personRun(candidate), taskStamp); continue; } if (!claimed) { if (slots.full) break; noteDeclinedAgedClaim(candidate, "claim_declined"); - backOffClaim(candidate); + backOffClaim(candidate, dispatcher.personRun(candidate), taskStamp); continue; } claimedRuns.push(claimed); @@ -2586,6 +2620,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende retryOfRunId: heartbeatRuns.retryOfRunId, scheduledRetryAt: heartbeatRuns.scheduledRetryAt, scheduledRetryAttempt: heartbeatRuns.scheduledRetryAttempt, + scheduledRetryReason: heartbeatRuns.scheduledRetryReason, issueId: contextText("issueId"), wakeReason: contextText("wakeReason"), previousRunId: contextText("previousRunId"), @@ -5741,6 +5776,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende createNativeStatusWakeBatch, startNextQueuedRunForAgent, countRunningRunsForAgent, + releaseClaimBackoffsForTask, releaseRunClaimedJustBeforeSuppression, claimQueuedRun, withChatControlRecoveryGate, diff --git a/server/src/services/heartbeat/run-slot-admission.ts b/server/src/services/heartbeat/run-slot-admission.ts index 8034ffa0e12..ec70cc2ec64 100644 --- a/server/src/services/heartbeat/run-slot-admission.ts +++ b/server/src/services/heartbeat/run-slot-admission.ts @@ -1,5 +1,6 @@ import { agents, heartbeatRuns, type Db } from "@paperclipai/db"; -import { and, eq, sql } from "drizzle-orm"; +import { isUuidLike } from "@paperclipai/shared"; +import { and, eq, notInArray, sql } from "drizzle-orm"; import { maxConcurrentRunsSql } from "./timer-policy.js"; export class AgentRunSlotsFull extends Error { @@ -9,13 +10,19 @@ export class AgentRunSlotsFull extends Error { } } -export async function assertAgentRunSlotFree(tx: Pick, companyId: string, agentId: string) { - const [slots] = await tx.select({ - limit: maxConcurrentRunsSql(), - running: sql`(select count(*)::int from ${heartbeatRuns} where ${heartbeatRuns.companyId} = ${companyId}::uuid and ${heartbeatRuns.agentId} = ${agentId}::uuid and ${heartbeatRuns.status} = 'running')`, - }).from(agents).where(and(eq(agents.id, agentId), eq(agents.companyId, companyId))); - if (!slots) return; - const running = Number(slots.running); - const limit = Number(slots.limit); - if (running >= limit) throw new AgentRunSlotsFull(agentId, running, limit); +type ClaimingRun = Pick; + +export async function assertAgentRunSlotFree(tx: Pick, run: ClaimingRun) { + const previousRunId = (run.contextSnapshot as Record | null)?.previousRunId; + const replaced = [run.id, run.retryOfRunId, typeof previousRunId === "string" ? previousRunId : null] + .filter((id): id is string => typeof id === "string" && isUuidLike(id)); + const [running] = await tx.select({ count: sql`count(*)::int` }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.agentId, run.agentId), eq(heartbeatRuns.status, "running"), + notInArray(heartbeatRuns.id, replaced))); + const [agent] = await tx.select({ limit: maxConcurrentRunsSql() }).from(agents) + .where(and(eq(agents.id, run.agentId), eq(agents.companyId, run.companyId))); + if (!agent) return; + const count = Number(running?.count ?? 0); + const limit = Number(agent.limit); + if (count >= limit) throw new AgentRunSlotsFull(run.agentId, count, limit); } diff --git a/server/src/services/native-runtime/native-review-dispatch.ts b/server/src/services/native-runtime/native-review-dispatch.ts index e38367c1d86..13eda8f9fc7 100644 --- a/server/src/services/native-runtime/native-review-dispatch.ts +++ b/server/src/services/native-runtime/native-review-dispatch.ts @@ -38,7 +38,7 @@ export async function claimQueuedNativeReviewRun(db: Db, input: { if ((run.contextSnapshot as Record | null)?.issueId !== issueId) return null; await withExternalAdmissionGuard(tx, run.companyId, run.agentId, async () => { await assertNoExternalOverlap(tx, run.companyId, run.agentId); - await assertAgentRunSlotFree(tx, run.companyId, run.agentId); + await assertAgentRunSlotFree(tx, run); }); const locked = await claimNativeReviewExecutionLock(tx as unknown as Db, { companyId: run.companyId, issueId, agentId: run.agentId, runId: run.id,