fix(heartbeat): let successors start after their predecessor and stop chaining passes on refused runs

Back off a refused claim only when the run has waited past the aging limit,
end a backoff when the run's task changes or a run of that task ends, and
count a successor's slot without the predecessor it replaces. A pass that
reaches its claim budget hands the runs it tried to one follow-up pass, so
refused person runs cannot hold back runnable work or chain passes. A trigger
after a pass's deadline also ends that pass, measured on a monotonic clock
from when it started.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-13 12:48:26 -06:00
1 parent 68e64a2de9
commit 3d48c4bed2
7 files changed
+155 -37

No files matched your search

@@ -234,4 +234,56 @@ suite("queued-run dispatch under a trigger burst", () => {
await heartbeat.drainActiveRunExecutions();
}
}, 60_000);
async function passesSettle() {
let last = -1;
await vi.waitFor(() => {
const settled = passes.started === last && !passes.active;
last = passes.started;
expect(settled).toBe(true);
}, { timeout: 30_000, interval: 1_000 });
return passes.started;
}
it("stops chaining passes once every refused run was tried", async () => {
passes.started = 0; passes.active = 0; passes.peak = 0;
const agentId = await seedAgent();
const otherTask = await task(agentId);
for (let index = 0; index < 60; index += 1) {
await queuedContinuation(agentId, { minutesAgo: 180 - index, wakeReason: "issue_continuation_needed", sourceIssueId: otherTask.id });
}
const heartbeat = heartbeatService(db);
try {
await heartbeat.resumeQueuedRunsForAgent(companyId, agentId);
expect(await passesSettle()).toBe(3);
expect(passes.peak).toBe(1);
} finally {
await heartbeat.drainActiveRunExecutions();
}
}, 120_000);
it("starts runnable work queued behind more refused person runs than one pass tries", async () => {
held.reset();
passes.started = 0; passes.active = 0; passes.peak = 0;
const agentId = await seedAgent();
const otherTask = await task(agentId);
for (let index = 0; index < 30; index += 1) {
await queuedContinuation(agentId, { minutesAgo: 90 - index, wakeReason: "task_retry", sourceIssueId: otherTask.id });
}
const runnable: string[] = [];
for (let index = 0; index < 3; index += 1) {
runnable.push((await queuedContinuation(agentId, { minutesAgo: 30 - index, wakeReason: "issue_assigned" })).id);
}
const heartbeat = heartbeatService(db);
try {
await heartbeat.resumeQueuedRunsForAgent(companyId, agentId);
expect(await passesSettle()).toBeLessThanOrEqual(3);
const started = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns)
.where(sql`${heartbeatRuns.id} in (${sql.join(runnable.map(id => sql`${id}::uuid`), sql`, `)}) and ${heartbeatRuns.startedAt} is not null`);
expect(started).toHaveLength(runnable.length);
} finally {
held.release();
await heartbeat.drainActiveRunExecutions();
}
}, 120_000);
});
@@ -95,6 +95,21 @@ describe("agent dispatch passes", () => {
expect(await dispatchForAgent(agentId, spec(false, async () => undefined))).toBe(3);
});
it("lets a trigger stop a hung pass by the pass's start time when its deadline timer has not run", async () => {
vi.useFakeTimers({ toFake: ["performance"] });
const agentId = randomUUID();
const { stats, spec } = meter();
let hungSignal: AbortSignal | null = null;
void dispatchForAgent(agentId, spec(false, (signal) => {
hungSignal = signal;
return new Promise<void>(() => undefined);
}));
vi.advanceTimersByTime(AGENT_DISPATCH_PASS_DEADLINE_MS);
expect(await dispatchForAgent(agentId, spec(false, async () => undefined))).toBe(2);
expect(hungSignal!.aborted).toBe(true);
expect(stats.passes).toBe(2);
});
it("does not wait on itself when a pass triggers its own agent", async () => {
const agentId = randomUUID();
const { stats, spec } = meter();
+13 -6
View File
@@ -13,7 +13,7 @@ export type AgentDispatchPass<R, T> = {
type Deferred<T> = { promise: Promise<T>; resolve(value: T): void; reject(err: unknown): void };
type FollowUp = { request: unknown; spec: AgentDispatchPass<unknown, unknown>; done: Deferred<unknown> };
type AgentDispatchState = { followUp: FollowUp | null };
type AgentDispatchState = { followUp: FollowUp | null; startedAtMs: number; expire(): void };
const dispatchStates = new Map<string, AgentDispatchState>();
const dispatchingAgent = new AsyncLocalStorage<string>();
@@ -38,7 +38,11 @@ export function outsideAgentDispatch<T>(fn: () => T): T {
}
export function dispatchForAgent<R, T>(agentId: string, spec: AgentDispatchPass<R, T>): Promise<T> {
const state = dispatchStates.get(agentId);
let state = dispatchStates.get(agentId);
if (state && performance.now() - state.startedAtMs >= AGENT_DISPATCH_PASS_DEADLINE_MS) {
state.expire();
state = dispatchStates.get(agentId);
}
if (!state) return startPass(agentId, spec, spec.request);
if (state.followUp) {
state.followUp.request = spec.merge(state.followUp.request as R, spec.request);
@@ -51,10 +55,10 @@ export function dispatchForAgent<R, T>(agentId: string, spec: AgentDispatchPass<
}
function startPass<R, T>(agentId: string, spec: AgentDispatchPass<R, T>, request: R): Promise<T> {
const state: AgentDispatchState = { followUp: null };
dispatchStates.set(agentId, state);
const controller = new AbortController();
const expired = deferred<void>();
const state: AgentDispatchState = { followUp: null, startedAtMs: performance.now(), expire: () => expire() };
dispatchStates.set(agentId, state);
let released = false;
const release = () => {
if (released) return;
@@ -64,13 +68,16 @@ function startPass<R, T>(agentId: string, spec: AgentDispatchPass<R, T>, request
state.followUp = null;
if (next) startPass(agentId, next.spec, next.request).then(next.done.resolve, next.done.reject);
};
const deadline = setTimeout(() => {
const expire = () => {
if (released) return;
clearTimeout(deadline);
logger.warn({ agentId, deadlineMs: AGENT_DISPATCH_PASS_DEADLINE_MS },
"queued-run dispatch pass passed its deadline; it stops claiming and the next pass may start");
controller.abort();
release();
expired.resolve();
}, AGENT_DISPATCH_PASS_DEADLINE_MS);
};
const deadline = setTimeout(expire, AGENT_DISPATCH_PASS_DEADLINE_MS);
deadline.unref?.();
const pass = dispatchingAgent.run(agentId, () => spec.run(request, controller.signal)).then(
async ({ value, settle }) => {
+1
View File
@@ -4388,6 +4388,7 @@ export function heartbeatService(
logger.warn({ err, runId: run.id }, "failed to deliver settled tool reviews after execution cleanup");
});
}
heartbeatQueue.releaseClaimBackoffsForTask(latestRun ?? run);
await startNextQueuedRunForAgent(run.agentId);
if (latestRun && isHeartbeatRunTerminalStatus(latestRun.status)) {
await notifyOtherAgentsQueuedOnSettledIssue(db, latestRun).catch(err => {
+56 -20
View File
@@ -216,8 +216,9 @@ import { crashPoint } from "@paperclipai/shared/crash-points";
type HeartbeatRun = typeof heartbeatRuns.$inferSelect;
type QueuedRunCandidate = Pick<HeartbeatRun, "id" | "companyId" | "agentId" | "status" | "invocationSource" | "createdAt"
| "startedAt" | "wakeupRequestId" | "retryOfRunId" | "scheduledRetryAt" | "scheduledRetryAttempt" | "contextSnapshot">;
type QueuedDispatchRequest = { attachSetupRequest: boolean; retryRefused: boolean };
| "startedAt" | "wakeupRequestId" | "retryOfRunId" | "scheduledRetryAt" | "scheduledRetryAttempt" | "scheduledRetryReason"
| "contextSnapshot">;
type QueuedDispatchRequest = { attachSetupRequest: boolean; retryRefused: boolean; triedRunIds: ReadonlySet<string> };
type HeartbeatRunState = ReturnType<typeof createHeartbeatRunState>;
type RunPreparation = ReturnType<typeof createHeartbeatRunPreparation>;
@@ -382,6 +383,7 @@ const CONVERSATION_COMMENT_KEY_PREFIX = "conversation-comment:";
const QUEUED_RUN_CLAIM_ATTEMPTS_PER_PASS = 24;
const QUEUED_RUN_CLAIM_BACKOFF_BASE_MS = 5_000;
const QUEUED_RUN_CLAIM_BACKOFF_MAX_MS = 60_000;
const QUEUED_PERSON_RUN_CLAIM_BACKOFF_MS = 2_000;
function dailyLimitConversationRefusal(cap: { reason: string; observed: number; limit: number }) {
// No receipt is written, so the person's message stays owed in the outbox
// until the limit resets (agent-conversations.ts conversationDeliveryHold).
@@ -1593,7 +1595,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
// live-comment discard gate below.
await withExternalAdmissionGuard(tx, run.companyId, run.agentId, async () => {
await assertNoExternalOverlap(tx, run.companyId, run.agentId);
await assertAgentRunSlotFree(tx, run.companyId, run.agentId);
await assertAgentRunSlotFree(tx, run);
});
await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId });
const [claimedRun] = await tx
@@ -1692,7 +1694,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
await withExternalAdmissionGuard(tx, run.companyId, run.agentId, async () => {
await assertNoExternalOverlap(tx, run.companyId, run.agentId);
await assertAgentRunSlotFree(tx, run.companyId, run.agentId);
await assertAgentRunSlotFree(tx, run);
});
await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId });
await tx
@@ -1815,7 +1817,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
if (!lockedRun || lockedRun.status !== "queued") return null;
await withExternalAdmissionGuard(claimTx, run.companyId, run.agentId, async () => {
await assertNoExternalOverlap(claimTx, run.companyId, run.agentId);
await assertAgentRunSlotFree(claimTx, run.companyId, run.agentId);
await assertAgentRunSlotFree(claimTx, run);
});
await notifyChatRunMilestones(claimTx, { companyId: run.companyId, runId: run.id, issueId: issueId ?? undefined });
const claimedRun = await claimTx.update(heartbeatRuns).set(claimValues).where(and(
@@ -2370,16 +2372,36 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
"long-queued heartbeat run was offered a free slot but could not be claimed; it stays queued and the next oldest run takes its turn");
}
const claimBackoffs = new Map<string, { agentId: string; attempts: number; until: number }>();
const claimBackoffs = new Map<string, { agentId: string; issueId: string | null; attempts: number; until: number;
taskStamp: string | null }>();
function claimBackedOff(runId: string, now: Date) {
return (claimBackoffs.get(runId)?.until ?? 0) > now.getTime();
function claimBackedOff(runId: string, now: Date, taskStamp: string | null) {
const backoff = claimBackoffs.get(runId);
return Boolean(backoff && backoff.until > now.getTime() && backoff.taskStamp === taskStamp);
}
function backOffClaim(run: QueuedRunCandidate) {
function backOffClaim(run: QueuedRunCandidate, personRun: boolean, taskStamp: string | null) {
if (!hasQueuedRunAged(run, new Date())) return;
const attempts = (claimBackoffs.get(run.id)?.attempts ?? 0) + 1;
const delayMs = Math.min(QUEUED_RUN_CLAIM_BACKOFF_BASE_MS * 2 ** (attempts - 1), QUEUED_RUN_CLAIM_BACKOFF_MAX_MS);
claimBackoffs.set(run.id, { agentId: run.agentId, attempts, until: Date.now() + delayMs });
const delayMs = personRun ? QUEUED_PERSON_RUN_CLAIM_BACKOFF_MS
: Math.min(QUEUED_RUN_CLAIM_BACKOFF_BASE_MS * 2 ** (attempts - 1), QUEUED_RUN_CLAIM_BACKOFF_MAX_MS);
claimBackoffs.set(run.id, { agentId: run.agentId, issueId: readNonEmptyString(parseObject(run.contextSnapshot).issueId),
attempts, until: Date.now() + delayMs, taskStamp });
}
function releaseClaimBackoffsForTask(run: Pick<HeartbeatRun, "issueId" | "contextSnapshot">) {
const issueId = run.issueId ?? readNonEmptyString(parseObject(run.contextSnapshot).issueId);
if (!issueId) return;
for (const [runId, backoff] of claimBackoffs) {
if (backoff.issueId === issueId) claimBackoffs.delete(runId);
}
}
function queuedRunTaskStamp(run: QueuedRunCandidate,
issueById: Map<string, { status: string; updatedAt: Date; executionRunId: string | null }>) {
const issueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId);
const issue = issueId ? issueById.get(issueId) : null;
return issue ? `${issue.status}:${issue.updatedAt.getTime()}:${issue.executionRunId ?? ""}` : null;
}
function forgetSettledClaimBackoffs(agentId: string, queuedRuns: QueuedRunCandidate[]) {
@@ -2399,13 +2421,14 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
agent: typeof agents.$inferSelect,
queuedRuns: QueuedRunCandidate[],
dependencyReadiness: Map<string, { isDependencyReady: boolean }>,
issueById: Map<string, { status: string; priority: string | null }>,
issueById: Map<string, { status: string; priority: string | null; updatedAt: Date; executionRunId: string | null }>,
) {
const now = new Date();
pruneDeclinedAgedClaims(now);
const runById = new Map(queuedRuns.map(run => [run.id, run]));
const { facts, dispatchInput } = await queuedRunDispatchFacts(db, {
agent, queuedRuns, dependencyReadiness, issueById, now,
shareDeferred: runId => declinedAgedClaimAt.has(runId) || claimBackedOff(runId, now) });
shareDeferred: runId => declinedAgedClaimAt.has(runId) || claimBackedOff(runId, now, queuedRunTaskStamp(runById.get(runId)!, issueById)) });
const dispatcher = createQueuedRunDispatcher(facts, dispatchInput);
const byRun = new Map(facts.map(entry => [entry.run, entry]));
return {
@@ -2420,12 +2443,17 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
}
function startNextQueuedRunForAgent(agentId: string, attachSetupRequest = false, retryRefused = false) {
return requestQueuedDispatch(agentId, { attachSetupRequest, retryRefused, triedRunIds: new Set() });
}
function requestQueuedDispatch(agentId: string, request: QueuedDispatchRequest) {
return dispatchForAgent<QueuedDispatchRequest, Array<typeof heartbeatRuns.$inferSelect>>(agentId, {
request: { attachSetupRequest, retryRefused },
request,
idle: [],
merge: (current, next) => ({
attachSetupRequest: current.attachSetupRequest || next.attachSetupRequest,
retryRefused: current.retryRefused || next.retryRefused,
triedRunIds: new Set([...current.triedRunIds, ...next.triedRunIds]),
}),
run: (request, signal) => runQueuedDispatchPass(agentId, request, signal),
track: (pass) => {
@@ -2436,7 +2464,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
});
}
async function runQueuedDispatchPass(agentId: string, { attachSetupRequest, retryRefused }: QueuedDispatchRequest, signal: AbortSignal) {
async function runQueuedDispatchPass(agentId: string, { attachSetupRequest, retryRefused, triedRunIds }: QueuedDispatchRequest,
signal: AbortSignal) {
// Cancelled after the pass ends: cancelRunInternal promotes the agent's
// next queued run, which asks for another pass of this agent.
const rejectedClaims: Array<{ run: typeof heartbeatRuns.$inferSelect; err: HttpError }> = [];
@@ -2495,7 +2524,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
return issueId ? [issueId] : [];
}))];
const issueRows = queuedIssueIds.length > 0
? await db.select({ id: issues.id, status: issues.status, priority: issues.priority }).from(issues)
? await db.select({ id: issues.id, status: issues.status, priority: issues.priority, updatedAt: issues.updatedAt,
executionRunId: issues.executionRunId }).from(issues)
.where(and(eq(issues.companyId, agent.companyId), inArray(issues.id, queuedIssueIds)))
: [];
const issueById = new Map(issueRows.map((row) => [row.id, row]));
@@ -2505,14 +2535,18 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
const claimedRuns: Array<typeof heartbeatRuns.$inferSelect> = [];
const slots = { full: false };
let claimAttempts = 0;
const triedThisPass: string[] = [];
for (let candidate = dispatcher.next(); candidate; candidate = dispatcher.next()) {
if (claimedRuns.length >= availableSlots || slots.full || isRunAdmissionClosed() || signal.aborted) break;
if (!retryRefused && !dispatcher.personRun(candidate) && claimBackedOff(candidate.id, new Date())) continue;
const taskStamp = queuedRunTaskStamp(candidate, issueById);
if (triedRunIds.has(candidate.id) || (!retryRefused && claimBackedOff(candidate.id, new Date(), taskStamp))) continue;
if (claimAttempts >= QUEUED_RUN_CLAIM_ATTEMPTS_PER_PASS) {
void startNextQueuedRunForAgent(agentId);
void requestQueuedDispatch(agentId, { attachSetupRequest: false, retryRefused,
triedRunIds: new Set([...triedRunIds, ...triedThisPass]) });
break;
}
claimAttempts += 1;
triedThisPass.push(candidate.id);
const [queuedRun] = await db.select().from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, candidate.id), eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.status, "queued")));
if (!queuedRun) continue;
@@ -2530,13 +2564,13 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
"queued heartbeat run claim was rejected; leaving it queued for the next recovery pass",
);
noteDeclinedAgedClaim(candidate, err.message);
backOffClaim(candidate);
backOffClaim(candidate, dispatcher.personRun(candidate), taskStamp);
continue;
}
if (!claimed) {
if (slots.full) break;
noteDeclinedAgedClaim(candidate, "claim_declined");
backOffClaim(candidate);
backOffClaim(candidate, dispatcher.personRun(candidate), taskStamp);
continue;
}
claimedRuns.push(claimed);
@@ -2586,6 +2620,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
retryOfRunId: heartbeatRuns.retryOfRunId,
scheduledRetryAt: heartbeatRuns.scheduledRetryAt,
scheduledRetryAttempt: heartbeatRuns.scheduledRetryAttempt,
scheduledRetryReason: heartbeatRuns.scheduledRetryReason,
issueId: contextText("issueId"),
wakeReason: contextText("wakeReason"),
previousRunId: contextText("previousRunId"),
@@ -5741,6 +5776,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
createNativeStatusWakeBatch,
startNextQueuedRunForAgent,
countRunningRunsForAgent,
releaseClaimBackoffsForTask,
releaseRunClaimedJustBeforeSuppression,
claimQueuedRun,
withChatControlRecoveryGate,
@@ -1,5 +1,6 @@
import { agents, heartbeatRuns, type Db } from "@paperclipai/db";
import { and, eq, sql } from "drizzle-orm";
import { isUuidLike } from "@paperclipai/shared";
import { and, eq, notInArray, sql } from "drizzle-orm";
import { maxConcurrentRunsSql } from "./timer-policy.js";
export class AgentRunSlotsFull extends Error {
@@ -9,13 +10,19 @@ export class AgentRunSlotsFull extends Error {
}
}
export async function assertAgentRunSlotFree(tx: Pick<Db, "select">, companyId: string, agentId: string) {
const [slots] = await tx.select({
limit: maxConcurrentRunsSql(),
running: sql<number>`(select count(*)::int from ${heartbeatRuns} where ${heartbeatRuns.companyId} = ${companyId}::uuid and ${heartbeatRuns.agentId} = ${agentId}::uuid and ${heartbeatRuns.status} = 'running')`,
}).from(agents).where(and(eq(agents.id, agentId), eq(agents.companyId, companyId)));
if (!slots) return;
const running = Number(slots.running);
const limit = Number(slots.limit);
if (running >= limit) throw new AgentRunSlotsFull(agentId, running, limit);
type ClaimingRun = Pick<typeof heartbeatRuns.$inferSelect, "id" | "companyId" | "agentId" | "retryOfRunId" | "contextSnapshot">;
export async function assertAgentRunSlotFree(tx: Pick<Db, "select">, run: ClaimingRun) {
const previousRunId = (run.contextSnapshot as Record<string, unknown> | null)?.previousRunId;
const replaced = [run.id, run.retryOfRunId, typeof previousRunId === "string" ? previousRunId : null]
.filter((id): id is string => typeof id === "string" && isUuidLike(id));
const [running] = await tx.select({ count: sql<number>`count(*)::int` }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.agentId, run.agentId), eq(heartbeatRuns.status, "running"),
notInArray(heartbeatRuns.id, replaced)));
const [agent] = await tx.select({ limit: maxConcurrentRunsSql() }).from(agents)
.where(and(eq(agents.id, run.agentId), eq(agents.companyId, run.companyId)));
if (!agent) return;
const count = Number(running?.count ?? 0);
const limit = Number(agent.limit);
if (count >= limit) throw new AgentRunSlotsFull(run.agentId, count, limit);
}
@@ -38,7 +38,7 @@ export async function claimQueuedNativeReviewRun(db: Db, input: {
if ((run.contextSnapshot as Record<string, unknown> | null)?.issueId !== issueId) return null;
await withExternalAdmissionGuard(tx, run.companyId, run.agentId, async () => {
await assertNoExternalOverlap(tx, run.companyId, run.agentId);
await assertAgentRunSlotFree(tx, run.companyId, run.agentId);
await assertAgentRunSlotFree(tx, run);
});
const locked = await claimNativeReviewExecutionLock(tx as unknown as Db, {
companyId: run.companyId, issueId, agentId: run.agentId, runId: run.id,