fix(heartbeat): run one dispatch pass per agent, back off refused claims, and enforce the run limit in the claim

Triggers during a pass fold into one follow-up pass instead of stacking past
the 30 s stale lock. Each claim checks the agent's running count against
maxConcurrentRuns under the agent admission lock, so overlapping passes and a
second server cannot run an agent past its limit. A refused claim waits a
bounded backoff before an event-driven pass offers it again; sweeps and a
person's runs ignore it. The pass reads only the columns it needs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-13 11:13:27 -06:00
1 parent 426e8a35dc
commit 68e64a2de9
8 files changed
+675 -173

No files matched your search

@@ -1,30 +0,0 @@
import { randomUUID } from "node:crypto";
import { afterEach, describe, expect, it, vi } from "vitest";
import { withAgentStartLock } from "../services/agent-start-lock.ts";
describe("heartbeat agent start lock", () => {
afterEach(() => {
vi.useRealTimers();
});
it("does not let a stale start lock freeze later queued-run starts", async () => {
vi.useFakeTimers();
const agentId = randomUUID();
const firstStart = vi.fn(() => new Promise<void>(() => undefined));
const secondStart = vi.fn(async () => "started");
void withAgentStartLock(agentId, firstStart);
await Promise.resolve();
expect(firstStart).toHaveBeenCalledTimes(1);
const secondStartResult = withAgentStartLock(agentId, secondStart);
await Promise.resolve();
expect(secondStart).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(30_000);
await expect(secondStartResult).resolves.toBe("started");
expect(secondStart).toHaveBeenCalledTimes(1);
});
});
@@ -0,0 +1,237 @@
import { randomUUID } from "node:crypto";
import { eq, sql } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
agentWakeupRequests, agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issues,
} from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { heartbeatService } from "../services/heartbeat.ts";
import { runningProcesses } from "../adapters/index.ts";
import { logger } from "../middleware/logger.js";
import { isAgentDispatchActive } from "../services/agent-start-lock.js";
const held = vi.hoisted(() => {
const state = {
release: () => undefined as void,
released: Promise.resolve(),
reset() {
state.released = new Promise<void>((resolve) => { state.release = resolve; });
},
};
return state;
})
const passes = vi.hoisted(() => ({ started: 0, active: 0, peak: 0 }));
vi.mock("../adapters/index.ts", async () => {
const actual = await vi.importActual<typeof import("../adapters/index.ts")>("../adapters/index.ts");
return {
...actual,
getServerAdapter: vi.fn(() => ({
supportsLocalAgentJwt: false,
execute: async () => {
await held.released;
return { exitCode: 0, signal: null, timedOut: false, errorMessage: null, summary: "done", provider: "test", model: "test" };
},
})),
};
});
vi.mock("../services/agent-start-lock.js", async () => {
const actual = await vi.importActual<typeof import("../services/agent-start-lock.js")>("../services/agent-start-lock.js");
const measure = async <T>(pass: () => Promise<T>) => {
passes.started += 1;
passes.active += 1;
passes.peak = Math.max(passes.peak, passes.active);
try {
return await pass();
} finally {
passes.active -= 1;
}
};
return {
...actual,
dispatchForAgent: (agentId: string, spec: Parameters<typeof actual.dispatchForAgent>[1]) =>
actual.dispatchForAgent(agentId, { ...spec, run: (request, signal) => measure(() => spec.run(request, signal)) }),
};
});
const support = await getEmbeddedPostgresTestSupport();
const suite = support.supported ? describe : describe.skip;
const CAP = 4;
const UNCLAIMABLE = 12;
const CLAIMABLE = 3 * CAP;
const TRIGGERS = 40;
suite("queued-run dispatch under a trigger burst", () => {
let db!: ReturnType<typeof createDb>;
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
let companyId = "", userId = "";
beforeAll(async () => {
temporary = await startEmbeddedPostgresTestDatabase("paperclip-queued-dispatch-stacking-");
db = createDb(temporary.connectionString);
await db.execute(sql`create table running_peaks (agent_id uuid not null, running int not null)`);
await db.execute(sql`create function record_running_peak() returns trigger language plpgsql as $$
begin
insert into running_peaks select new.agent_id, count(*) from heartbeat_runs where agent_id = new.agent_id and status = 'running';
return new;
end $$`);
await db.execute(sql`create trigger record_running_peak after update of status on heartbeat_runs
for each row when (new.status = 'running') execute function record_running_peak()`);
}, 60_000);
afterEach(async () => {
vi.restoreAllMocks();
held.release();
runningProcesses.clear();
await db.execute(sql`truncate table ${companies} cascade`);
});
afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); });
async function seedAgent() {
userId = randomUUID();
companyId = randomUUID();
const agentId = randomUUID();
await db.insert(authUsers).values({ id: userId, name: "Board", email: `${userId}@example.test`,
emailVerified: true, createdAt: new Date(), updatedAt: new Date() });
await db.insert(companies).values({ id: companyId, name: "Burst", issuePrefix: `B${companyId.replaceAll("-", "").slice(0, 6)}`,
requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: userId });
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, status: "active", membershipRole: "owner" });
await db.insert(agents).values({ id: agentId, companyId, name: "Codie", role: "engineer", status: "idle",
adapterType: "codex_local", adapterConfig: {},
runtimeConfig: { heartbeat: { enabled: true, wakeOnDemand: true, maxConcurrentRuns: CAP } }, permissions: {} });
return agentId;
}
async function task(agentId: string) {
const [row] = await db.insert(issues).values({ companyId, title: "Continue the work", status: "in_progress", assigneeAgentId: agentId }).returning();
return row!;
}
async function finishedRun(agentId: string, issueId: string, minutesAgo: number) {
const finishedAt = new Date(Date.now() - minutesAgo * 60_000);
const [row] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId, invocationSource: "assignment",
status: "failed", errorCode: "server_shutdown_interrupted", startedAt: new Date(finishedAt.getTime() - 60_000), finishedAt,
createdAt: new Date(finishedAt.getTime() - 120_000), contextSnapshot: { issueId, wakeReason: "issue_assigned" } }).returning();
return row!;
}
async function queuedContinuation(agentId: string, input: { minutesAgo: number; wakeReason: string; sourceIssueId?: string }) {
const issue = await task(agentId);
const source = await finishedRun(agentId, input.sourceIssueId ?? issue.id, input.minutesAgo + 5);
const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "automation", triggerDetail: "system",
reason: input.wakeReason, status: "queued", requestedByActorType: "system", requestedByActorId: null,
payload: { issueId: issue.id } }).returning();
const [row] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: issue.id, invocationSource: "automation",
triggerDetail: "system", status: "queued", retryOfRunId: source.id, wakeupRequestId: wake!.id,
createdAt: new Date(Date.now() - input.minutesAgo * 60_000),
contextSnapshot: { issueId: issue.id, taskId: issue.id, wakeReason: input.wakeReason, previousRunId: source.id,
source: "issue.assigned_work_invariant", paperclipWorkspace: { notes: "x".repeat(16_000) } } }).returning();
await db.update(agentWakeupRequests).set({ runId: row!.id }).where(eq(agentWakeupRequests.id, wake!.id));
return row!;
}
const peakRunning = async (agentId: string) => Number((await db.execute(sql`select coalesce(max(running), 0) as peak
from running_peaks where agent_id = ${agentId}::uuid`))[0]!.peak);
it("runs one pass at a time for a burst of triggers, offers a refused run once, and never runs past the agent's limit", async () => {
held.reset();
passes.started = 0; passes.active = 0; passes.peak = 0;
const agentId = await seedAgent();
const otherTask = await task(agentId);
const unclaimable = new Set<string>();
for (let index = 0; index < UNCLAIMABLE; index += 1) {
unclaimable.add((await queuedContinuation(agentId, { minutesAgo: 180 - index, wakeReason: "issue_continuation_needed",
sourceIssueId: otherTask.id })).id);
}
for (let index = 0; index < CLAIMABLE; index += 1) {
await queuedContinuation(agentId, { minutesAgo: 30 - index, wakeReason: "issue_assigned" });
}
const realNow = Date.now.bind(Date);
let skewMs = 0;
vi.spyOn(Date, "now").mockImplementation(() => realNow() + skewMs);
const offers = new Map<string, number>();
let inFlightClaims = 0;
let peakInFlightClaims = 0;
const triggers: Array<Promise<unknown>> = [];
let burstFired = false;
const heartbeat = heartbeatService(db, {
beforeChatControlRecoveryCheck: async ({ runId, stage }) => {
if (stage !== "claim") return;
offers.set(runId, (offers.get(runId) ?? 0) + 1);
inFlightClaims += 1;
peakInFlightClaims = Math.max(peakInFlightClaims, inFlightClaims);
try {
if (!burstFired) {
burstFired = true;
for (let index = 0; index < TRIGGERS; index += 1) {
skewMs += 31_000;
triggers.push(heartbeat.resumeQueuedRunsForAgent(companyId, agentId));
await new Promise((resolve) => setTimeout(resolve, 100));
}
skewMs = 0;
}
await new Promise((resolve) => setTimeout(resolve, 25));
} finally {
inFlightClaims -= 1;
}
},
});
try {
await heartbeat.resumeQueuedRunsForAgent(companyId, agentId);
await Promise.all(triggers);
const refusedOffers = [...unclaimable].reduce((sum, id) => sum + (offers.get(id) ?? 0), 0);
expect({
passes: passes.started,
peakConcurrentPasses: passes.peak,
peakInFlightClaims,
refusedOffersPerTrigger: refusedOffers / (TRIGGERS + 1),
peakRunning: await peakRunning(agentId),
}).toEqual({
passes: 2,
peakConcurrentPasses: 1,
peakInFlightClaims: 1,
refusedOffersPerTrigger: UNCLAIMABLE / (TRIGGERS + 1),
peakRunning: CAP,
});
await vi.waitFor(() => {
expect(passes.active).toBe(0);
expect(isAgentDispatchActive(agentId)).toBe(false);
}, { timeout: 10_000 });
} finally {
held.release();
await heartbeat.drainActiveRunExecutions();
}
}, 120_000);
it.each(["declined", "rejected"] as const)("waits a bounded time before offering a %s run again, and offers it again once that time passes", async (refusal) => {
const agentId = await seedAgent();
if (refusal === "rejected") {
await db.update(companies).set({ defaultResponsibleUserId: null }).where(eq(companies.id, companyId));
await db.delete(companyMemberships).where(eq(companyMemberships.companyId, companyId));
}
const otherTask = await task(agentId);
const refused = await queuedContinuation(agentId, { minutesAgo: 180, wakeReason: "issue_continuation_needed", sourceIssueId: otherTask.id });
const warn = vi.spyOn(logger, "warn");
const offersOf = () => warn.mock.calls.filter(([fields, message]) => typeof message === "string"
&& message.startsWith("long-queued heartbeat run was offered a free slot")
&& (fields as { runId?: string }).runId === refused.id).length;
const heartbeat = heartbeatService(db);
try {
for (let index = 0; index < 10; index += 1) await heartbeat.resumeQueuedRunsForAgent(companyId, agentId);
expect(offersOf()).toBe(1);
await vi.waitFor(async () => {
await heartbeat.resumeQueuedRunsForAgent(companyId, agentId);
expect(offersOf()).toBe(2);
}, { timeout: 20_000, interval: 1_000 });
} finally {
await heartbeat.drainActiveRunExecutions();
}
}, 60_000);
});
@@ -0,0 +1,140 @@
import { randomUUID } from "node:crypto";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
AGENT_DISPATCH_PASS_DEADLINE_MS,
dispatchForAgent,
isAgentDispatchActive,
type AgentDispatchPass,
} from "./agent-start-lock.js";
function gate() {
let open!: () => void;
const opened = new Promise<void>((resolve) => { open = resolve; });
return { opened, open };
}
function meter() {
const stats = { passes: 0, active: 0, peak: 0, requests: [] as boolean[] };
const spec = (attach: boolean, body: (signal: AbortSignal) => Promise<void>): AgentDispatchPass<boolean, number> => ({
request: attach,
idle: -1,
merge: (current, next) => current || next,
async run(request, signal) {
stats.passes += 1;
stats.active += 1;
stats.peak = Math.max(stats.peak, stats.active);
stats.requests.push(request);
const pass = stats.passes;
try {
await body(signal);
} finally {
stats.active -= 1;
}
return { value: pass };
},
});
return { stats, spec };
}
describe("agent dispatch passes", () => {
afterEach(() => {
vi.useRealTimers();
});
it("runs one pass per agent and folds every trigger that arrives during it into one follow-up pass", async () => {
const agentId = randomUUID();
const { stats, spec } = meter();
const first = gate();
const firstPass = dispatchForAgent(agentId, spec(false, () => first.opened));
const burst = Array.from({ length: 50 }, (_, index) =>
dispatchForAgent(agentId, spec(index === 7, async () => undefined)));
expect(stats).toMatchObject({ passes: 1, peak: 1 });
first.open();
expect(await firstPass).toBe(1);
expect(new Set(await Promise.all(burst))).toEqual(new Set([2]));
expect(stats).toMatchObject({ passes: 2, peak: 1, requests: [false, true] });
expect(isAgentDispatchActive(agentId)).toBe(false);
});
it("keeps a second pass out while a slow pass runs past the old 30 second stale limit", async () => {
vi.useFakeTimers();
const agentId = randomUUID();
const { stats, spec } = meter();
const slow = gate();
const firstPass = dispatchForAgent(agentId, spec(false, () => slow.opened));
const triggers: Array<Promise<number>> = [];
for (let second = 0; second < 45; second += 1) {
await vi.advanceTimersByTimeAsync(1_000);
triggers.push(dispatchForAgent(agentId, spec(false, async () => undefined)));
}
expect(stats).toMatchObject({ passes: 1, peak: 1 });
slow.open();
await firstPass;
await Promise.all(triggers);
expect(stats).toMatchObject({ passes: 2, peak: 1 });
});
it("stops a hung pass at its deadline and lets the waiting trigger run without overlapping it", async () => {
vi.useFakeTimers();
const agentId = randomUUID();
const { stats, spec } = meter();
let hungSignal: AbortSignal | null = null;
void dispatchForAgent(agentId, spec(false, (signal) => {
hungSignal = signal;
return new Promise<void>(() => undefined);
}));
const waiting = dispatchForAgent(agentId, spec(false, async () => undefined));
await vi.advanceTimersByTimeAsync(AGENT_DISPATCH_PASS_DEADLINE_MS - 1);
expect(stats.passes).toBe(1);
expect(hungSignal!.aborted).toBe(false);
await vi.advanceTimersByTimeAsync(1);
expect(hungSignal!.aborted).toBe(true);
expect(await waiting).toBe(2);
expect(stats.passes).toBe(2);
expect(isAgentDispatchActive(agentId)).toBe(false);
expect(await dispatchForAgent(agentId, spec(false, async () => undefined))).toBe(3);
});
it("does not wait on itself when a pass triggers its own agent", async () => {
const agentId = randomUUID();
const { stats, spec } = meter();
let nested: Promise<number> | null = null;
const pass = dispatchForAgent(agentId, spec(false, async () => {
nested = dispatchForAgent(agentId, spec(false, async () => undefined));
expect(await nested).toBe(-1);
}));
expect(await pass).toBe(1);
await vi.waitFor(() => expect(stats.passes).toBe(2));
expect(stats.peak).toBe(1);
});
it("settles a pass's effects after the next pass may start", async () => {
const agentId = randomUUID();
const order: string[] = [];
const release = gate();
const pass = dispatchForAgent(agentId, {
request: null, idle: null, merge: () => null,
async run() {
await release.opened;
return {
value: null,
settle: async () => {
order.push(`settle:${isAgentDispatchActive(agentId)}`);
await dispatchForAgent(agentId, { request: null, idle: null, merge: () => null,
async run() { order.push("nested"); return { value: null }; } });
},
};
},
});
release.open();
await pass;
expect(order).toEqual(["settle:false", "nested"]);
});
it("passes a failed pass's error to its caller and frees the agent", async () => {
const agentId = randomUUID();
const { spec } = meter();
await expect(dispatchForAgent(agentId, spec(false, async () => { throw new Error("boom"); }))).rejects.toThrow("boom");
expect(isAgentDispatchActive(agentId)).toBe(false);
});
});
+81 -39
View File
@@ -1,48 +1,90 @@
import { AsyncLocalStorage } from "node:async_hooks";
import { logger } from "../middleware/logger.js";
const AGENT_START_LOCK_STALE_MS = 30_000;
const startLocksByAgent = new Map<string, { promise: Promise<void>; startedAtMs: number }>();
export const AGENT_DISPATCH_PASS_DEADLINE_MS = 60_000;
async function waitForAgentStartLock(agentId: string, lock: { promise: Promise<void>; startedAtMs: number }) {
const elapsedMs = Date.now() - lock.startedAtMs;
const remainingMs = AGENT_START_LOCK_STALE_MS - elapsedMs;
if (remainingMs <= 0) {
logger.warn({ agentId, staleMs: elapsedMs }, "agent start lock stale; continuing queued-run start");
return;
}
export type AgentDispatchPass<R, T> = {
request: R;
idle: T;
merge(current: R, next: R): R;
run(request: R, signal: AbortSignal): Promise<{ value: T; settle?: () => Promise<void> }>;
track?(pass: Promise<unknown>): void;
};
let timedOut = false;
let timeout: ReturnType<typeof setTimeout> | null = null;
await Promise.race([
lock.promise,
new Promise<void>((resolve) => {
timeout = setTimeout(() => {
timedOut = true;
resolve();
}, remainingMs);
}),
]);
if (timeout) clearTimeout(timeout);
type Deferred<T> = { promise: Promise<T>; resolve(value: T): void; reject(err: unknown): void };
type FollowUp = { request: unknown; spec: AgentDispatchPass<unknown, unknown>; done: Deferred<unknown> };
type AgentDispatchState = { followUp: FollowUp | null };
if (timedOut) {
logger.warn({ agentId, staleMs: AGENT_START_LOCK_STALE_MS }, "agent start lock timed out; continuing queued-run start");
}
const dispatchStates = new Map<string, AgentDispatchState>();
const dispatchingAgent = new AsyncLocalStorage<string>();
function deferred<T>(): Deferred<T> {
let resolve!: (value: T) => void;
let reject!: (err: unknown) => void;
const promise = new Promise<T>((res, rej) => {
resolve = res;
reject = rej;
});
promise.catch(() => undefined);
return { promise, resolve, reject };
}
export async function withAgentStartLock<T>(agentId: string, fn: () => Promise<T>) {
const previous = startLocksByAgent.get(agentId);
const waitForPrevious = previous ? waitForAgentStartLock(agentId, previous) : Promise.resolve();
const run = waitForPrevious.then(fn);
const marker = run.then(
() => undefined,
() => undefined,
export function isAgentDispatchActive(agentId: string) {
return dispatchStates.has(agentId);
}
export function outsideAgentDispatch<T>(fn: () => T): T {
return dispatchingAgent.exit(fn);
}
export function dispatchForAgent<R, T>(agentId: string, spec: AgentDispatchPass<R, T>): Promise<T> {
const state = dispatchStates.get(agentId);
if (!state) return startPass(agentId, spec, spec.request);
if (state.followUp) {
state.followUp.request = spec.merge(state.followUp.request as R, spec.request);
state.followUp.spec = spec as AgentDispatchPass<unknown, unknown>;
} else {
state.followUp = { request: spec.request, spec: spec as AgentDispatchPass<unknown, unknown>, done: deferred() };
}
if (dispatchingAgent.getStore() === agentId) return Promise.resolve(spec.idle);
return state.followUp.done.promise as Promise<T>;
}
function startPass<R, T>(agentId: string, spec: AgentDispatchPass<R, T>, request: R): Promise<T> {
const state: AgentDispatchState = { followUp: null };
dispatchStates.set(agentId, state);
const controller = new AbortController();
const expired = deferred<void>();
let released = false;
const release = () => {
if (released) return;
released = true;
if (dispatchStates.get(agentId) === state) dispatchStates.delete(agentId);
const next = state.followUp;
state.followUp = null;
if (next) startPass(agentId, next.spec, next.request).then(next.done.resolve, next.done.reject);
};
const deadline = setTimeout(() => {
logger.warn({ agentId, deadlineMs: AGENT_DISPATCH_PASS_DEADLINE_MS },
"queued-run dispatch pass passed its deadline; it stops claiming and the next pass may start");
controller.abort();
release();
expired.resolve();
}, AGENT_DISPATCH_PASS_DEADLINE_MS);
deadline.unref?.();
const pass = dispatchingAgent.run(agentId, () => spec.run(request, controller.signal)).then(
async ({ value, settle }) => {
clearTimeout(deadline);
release();
if (settle) await outsideAgentDispatch(settle);
return value;
},
(err: unknown) => {
clearTimeout(deadline);
release();
throw err;
},
);
startLocksByAgent.set(agentId, { promise: marker, startedAtMs: Date.now() });
try {
return await run;
} finally {
if (startLocksByAgent.get(agentId)?.promise === marker) {
startLocksByAgent.delete(agentId);
}
}
spec.track?.(Promise.race([pass.then(() => undefined, () => undefined), expired.promise]));
return pass;
}
+188 -101
View File
@@ -12,7 +12,7 @@ import { hasAutomaticInteractionDeliveryHold, INTERACTION_DELIVERY_DENIED, reche
import { deferredWorkIntentWait } from "../work-intent-outbox.js";
import { notifyIssueRecovery } from "../issue-recovery-notifications.js";
import { createHeartbeatTimerPolicy } from "./timer-policy.js";
import { QUEUED_RUN_AGING_MS, createQueuedRunDispatcher, hasQueuedRunAged } from "./queued-run-dispatch-order.js";
import { QUEUED_RUN_AGING_MS, createQueuedRunDispatcher, hasQueuedRunAged, queuedRunDispatchClass } from "./queued-run-dispatch-order.js";
import { queuedRunDispatchFacts } from "./queued-run-line.js";
import { notifyExecutionRunWork } from "../execution-control-notifications.js";
import { PersistedExecutionWaitRefusal, writeDeferredExecutionWait } from "../deferred-execution-wait.js";
@@ -188,7 +188,8 @@ import {
} from "../execution-workspace-policy.js";
import { readContinuationAttempt } from "../recovery/index.js";
import { allowsDependencyBlockedWake, extractWakeCommentIds, wakeChatConversationsConflict } from "../../modules/run-dispatch/index.js";
import { withAgentStartLock } from "../agent-start-lock.js";
import { dispatchForAgent, outsideAgentDispatch } from "../agent-start-lock.js";
import { AgentRunSlotsFull, assertAgentRunSlotFree } from "./run-slot-admission.js";
import {
evaluateAgentInvokability,
shouldCancelRunsForNonInvokableAgent,
@@ -214,6 +215,9 @@ import { hasRetainedExternalChatRetry, type PostCommitEffect as WakeQueuePostCom
import { crashPoint } from "@paperclipai/shared/crash-points";
type HeartbeatRun = typeof heartbeatRuns.$inferSelect;
type QueuedRunCandidate = Pick<HeartbeatRun, "id" | "companyId" | "agentId" | "status" | "invocationSource" | "createdAt"
| "startedAt" | "wakeupRequestId" | "retryOfRunId" | "scheduledRetryAt" | "scheduledRetryAttempt" | "contextSnapshot">;
type QueuedDispatchRequest = { attachSetupRequest: boolean; retryRefused: boolean };
type HeartbeatRunState = ReturnType<typeof createHeartbeatRunState>;
type RunPreparation = ReturnType<typeof createHeartbeatRunPreparation>;
@@ -375,6 +379,9 @@ export function shouldQueueFollowupForRunningIssueWake(input: {
/** Agent Chat outbox receipts. The outbox, not a skipped receipt, owns a capped message. */
const CONVERSATION_COMMENT_KEY_PREFIX = "conversation-comment:";
const QUEUED_RUN_CLAIM_ATTEMPTS_PER_PASS = 24;
const QUEUED_RUN_CLAIM_BACKOFF_BASE_MS = 5_000;
const QUEUED_RUN_CLAIM_BACKOFF_MAX_MS = 60_000;
function dailyLimitConversationRefusal(cap: { reason: string; observed: number; limit: number }) {
// No receipt is written, so the person's message stays owed in the outbox
// until the limit resets (agent-conversations.ts conversationDeliveryHold).
@@ -597,7 +604,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
async function listQueuedRunDependencyReadiness(
companyId: string,
queuedRuns: Array<typeof heartbeatRuns.$inferSelect>,
queuedRuns: Array<Pick<HeartbeatRun, "contextSnapshot">>,
) {
const issueIds = [
...new Set(
@@ -960,6 +967,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
companyAgents?: AgentOrgRow[],
deferredPostCommitEffects?: WakeQueuePostCommitEffect[],
deferredCancellations?: Array<() => Promise<void>>,
slots?: { full: boolean },
) {
if (run.status !== "queued") return run;
// Every start path ends here (dispatch passes, executeRun of a queued run,
@@ -1583,7 +1591,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
// envelope. Preserve their established claim path; only an
// explicitly bound queued-message envelope is subject to the
// live-comment discard gate below.
await withExternalAdmissionGuard(tx, run.companyId, run.agentId, () => assertNoExternalOverlap(tx, run.companyId, run.agentId));
await withExternalAdmissionGuard(tx, run.companyId, run.agentId, async () => {
await assertNoExternalOverlap(tx, run.companyId, run.agentId);
await assertAgentRunSlotFree(tx, run.companyId, run.agentId);
});
await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId });
const [claimedRun] = await tx
.update(heartbeatRuns)
@@ -1679,7 +1690,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
};
}
await withExternalAdmissionGuard(tx, run.companyId, run.agentId, () => assertNoExternalOverlap(tx, run.companyId, run.agentId));
await withExternalAdmissionGuard(tx, run.companyId, run.agentId, async () => {
await assertNoExternalOverlap(tx, run.companyId, run.agentId);
await assertAgentRunSlotFree(tx, run.companyId, run.agentId);
});
await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId });
await tx
.update(agentWakeupRequests)
@@ -1723,6 +1737,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
.catch((error) => {
if (isExternalChatWaitAuthorizationContention(error) || error instanceof RunAdmissionClosedDuringClaim)
return { kind: "stale" as const, run: null };
if (error instanceof AgentRunSlotsFull) {
if (slots) slots.full = true;
return { kind: "stale" as const, run: null };
}
throw error;
})
: null;
@@ -1795,7 +1813,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.agentId, run.agentId),
)).for("update");
if (!lockedRun || lockedRun.status !== "queued") return null;
await withExternalAdmissionGuard(claimTx, run.companyId, run.agentId, () => assertNoExternalOverlap(claimTx, run.companyId, run.agentId));
await withExternalAdmissionGuard(claimTx, run.companyId, run.agentId, async () => {
await assertNoExternalOverlap(claimTx, run.companyId, run.agentId);
await assertAgentRunSlotFree(claimTx, run.companyId, run.agentId);
});
await notifyChatRunMilestones(claimTx, { companyId: run.companyId, runId: run.id, issueId: issueId ?? undefined });
const claimedRun = await claimTx.update(heartbeatRuns).set(claimValues).where(and(
eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "queued"),
@@ -1806,6 +1827,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
});
}).catch((error: unknown) => {
if (error instanceof RunAdmissionClosedDuringClaim) return null;
if (error instanceof AgentRunSlotsFull) {
if (slots) slots.full = true;
return null;
}
throw error;
});
if (!claimed) return null;
@@ -2252,7 +2277,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
const agentIds = [...new Set(queuedRuns.map((r) => r.agentId))];
for (const agentId of agentIds) {
await startNextQueuedRunForAgent(agentId, true);
await startNextQueuedRunForAgent(agentId, true, true);
}
}
@@ -2336,7 +2361,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
const declinedAgedClaimAt = new Map<string, number>();
function noteDeclinedAgedClaim(run: typeof heartbeatRuns.$inferSelect, reason: string) {
function noteDeclinedAgedClaim(run: QueuedRunCandidate, reason: string) {
const now = Date.now();
if (!hasQueuedRunAged(run, new Date(now))) return;
declinedAgedClaimAt.set(run.id, now);
@@ -2345,6 +2370,25 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
"long-queued heartbeat run was offered a free slot but could not be claimed; it stays queued and the next oldest run takes its turn");
}
const claimBackoffs = new Map<string, { agentId: string; attempts: number; until: number }>();
function claimBackedOff(runId: string, now: Date) {
return (claimBackoffs.get(runId)?.until ?? 0) > now.getTime();
}
function backOffClaim(run: QueuedRunCandidate) {
const attempts = (claimBackoffs.get(run.id)?.attempts ?? 0) + 1;
const delayMs = Math.min(QUEUED_RUN_CLAIM_BACKOFF_BASE_MS * 2 ** (attempts - 1), QUEUED_RUN_CLAIM_BACKOFF_MAX_MS);
claimBackoffs.set(run.id, { agentId: run.agentId, attempts, until: Date.now() + delayMs });
}
function forgetSettledClaimBackoffs(agentId: string, queuedRuns: QueuedRunCandidate[]) {
const queued = new Set(queuedRuns.map(run => run.id));
for (const [runId, backoff] of claimBackoffs) {
if (backoff.agentId === agentId && !queued.has(runId)) claimBackoffs.delete(runId);
}
}
function pruneDeclinedAgedClaims(now: Date) {
for (const [id, at] of declinedAgedClaimAt) {
if (now.getTime() - at >= QUEUED_RUN_AGING_MS) declinedAgedClaimAt.delete(id);
@@ -2353,36 +2397,78 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
async function queuedRunDispatcherForAgent(
agent: typeof agents.$inferSelect,
queuedRuns: Array<typeof heartbeatRuns.$inferSelect>,
queuedRuns: QueuedRunCandidate[],
dependencyReadiness: Map<string, { isDependencyReady: boolean }>,
issueById: Map<string, { status: string; priority: string | null }>,
) {
const now = new Date();
pruneDeclinedAgedClaims(now);
const { facts, dispatchInput } = await queuedRunDispatchFacts(db, {
agent, queuedRuns, dependencyReadiness, issueById, now, shareDeferred: runId => declinedAgedClaimAt.has(runId) });
agent, queuedRuns, dependencyReadiness, issueById, now,
shareDeferred: runId => declinedAgedClaimAt.has(runId) || claimBackedOff(runId, now) });
const dispatcher = createQueuedRunDispatcher(facts, dispatchInput);
const byRun = new Map(facts.map(entry => [entry.run, entry]));
return {
next: () => dispatcher.next()?.run ?? null,
claimed: (run: typeof heartbeatRuns.$inferSelect) => { const entry = byRun.get(run); if (entry) dispatcher.claimed(entry); },
claimed: (run: QueuedRunCandidate) => { const entry = byRun.get(run); if (entry) dispatcher.claimed(entry); },
personRun: (run: QueuedRunCandidate) => {
const entry = byRun.get(run);
return Boolean(entry && queuedRunDispatchClass(entry, now) === "person");
},
personOrigin: (run: QueuedRunCandidate) => byRun.get(run)?.requestedByUser ?? false,
};
}
async function startNextQueuedRunForAgent(agentId: string, attachSetupRequest = false) {
// Null unless this server has not read the run-start hand-over recently.
const refreshing = refreshRunAdmission();
if (refreshing) await refreshing;
if (isRunAdmissionClosed() || (await getSchedulingSuppression()).suppressed) return [];
const cutoff = await getWorktreeExecutionCutoff();
// Cancelled after the start lock is released: cancelRunInternal promotes the
// agent's next queued run, which takes this same lock.
function startNextQueuedRunForAgent(agentId: string, attachSetupRequest = false, retryRefused = false) {
return dispatchForAgent<QueuedDispatchRequest, Array<typeof heartbeatRuns.$inferSelect>>(agentId, {
request: { attachSetupRequest, retryRefused },
idle: [],
merge: (current, next) => ({
attachSetupRequest: current.attachSetupRequest || next.attachSetupRequest,
retryRefused: current.retryRefused || next.retryRefused,
}),
run: (request, signal) => runQueuedDispatchPass(agentId, request, signal),
track: (pass) => {
const tracked = pass.then(() => undefined);
activeRunExecutionPromises.add(tracked);
void tracked.finally(() => activeRunExecutionPromises.delete(tracked));
},
});
}
async function runQueuedDispatchPass(agentId: string, { attachSetupRequest, retryRefused }: QueuedDispatchRequest, signal: AbortSignal) {
// Cancelled after the pass ends: cancelRunInternal promotes the agent's
// next queued run, which asks for another pass of this agent.
const rejectedClaims: Array<{ run: typeof heartbeatRuns.$inferSelect; err: HttpError }> = [];
const deferredPostCommitEffects: WakeQueuePostCommitEffect[] = [];
const deferredCancellations: Array<() => Promise<void>> = [];
let cancellationReason: string | undefined;
const settle = async () => {
// Dispatch promoted inputs after the pass has ended.
try {
await applyWakeQueuePostCommitEffects(deferredPostCommitEffects);
} finally {
if (cancellationReason) await cancelActiveForAgentInternal(agentId, cancellationReason);
await cancelRejectedQueuedRuns(rejectedClaims);
for (const cancel of deferredCancellations) {
await cancel().catch((err) => logger.warn({ err, agentId },
"queued heartbeat run refused by a pause hold could not be cancelled; the next claim retries it"));
}
}
};
try {
return { value: await claimQueuedRunsForAgent(), settle };
} catch (err) {
await settle();
throw err;
}
return withAgentStartLock(agentId, async () => {
async function claimQueuedRunsForAgent(): Promise<Array<typeof heartbeatRuns.$inferSelect>> {
// Null unless this server has not read the run-start hand-over recently.
const refreshing = refreshRunAdmission();
if (refreshing) await refreshing;
if (isRunAdmissionClosed() || (await getSchedulingSuppression()).suppressed) return [];
const cutoff = await getWorktreeExecutionCutoff();
const agent = await getAgent(agentId);
if (!agent) return [];
const invokability = await getAgentInvokability(agent);
@@ -2395,64 +2481,44 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
}
const policy = parseHeartbeatPolicy(agent);
const runningCount = await countRunningRunsForAgent(agentId);
const availableSlots = Math.max(
0,
policy.maxConcurrentRuns - runningCount,
);
const availableSlots = Math.max(0, policy.maxConcurrentRuns - runningCount);
if (availableSlots <= 0) return [];
const queuedRuns = await db
.select()
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.agentId, agentId),
eq(heartbeatRuns.status, "queued"),
cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined,
),
)
.orderBy(asc(heartbeatRuns.createdAt));
const queuedRuns = await listQueuedRunCandidates(agentId, cutoff);
forgetSettledClaimBackoffs(agentId, queuedRuns);
if (queuedRuns.length === 0) return [];
if (queuedRuns.every(run => !run.startedAt) && !await canStartNewExternalBotWork(agent)) return [];
const dependencyReadiness = await listQueuedRunDependencyReadiness(
agent.companyId,
queuedRuns,
);
const queuedIssueIds = [
...new Set(
queuedRuns
.map((run) =>
readNonEmptyString(parseObject(run.contextSnapshot).issueId),
)
.filter((issueId): issueId is string => Boolean(issueId)),
),
];
const issueRows = await db
.select({
id: issues.id,
status: issues.status,
priority: issues.priority,
})
.from(issues)
.where(
queuedIssueIds.length > 0
? and(
eq(issues.companyId, agent.companyId),
inArray(issues.id, queuedIssueIds),
)
: sql`false`,
);
const dependencyReadiness = await listQueuedRunDependencyReadiness(agent.companyId, queuedRuns);
const queuedIssueIds = [...new Set(queuedRuns.flatMap((run) => {
const issueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId);
return issueId ? [issueId] : [];
}))];
const issueRows = queuedIssueIds.length > 0
? await db.select({ id: issues.id, status: issues.status, priority: issues.priority }).from(issues)
.where(and(eq(issues.companyId, agent.companyId), inArray(issues.id, queuedIssueIds)))
: [];
const issueById = new Map(issueRows.map((row) => [row.id, row]));
const companyAgents = await listCompanyAgentOrgRows(agent.companyId);
const dispatcher = await queuedRunDispatcherForAgent(agent, queuedRuns, dependencyReadiness, issueById);
const claimedRuns: Array<typeof heartbeatRuns.$inferSelect> = [];
for (let queuedRun = dispatcher.next(); queuedRun; queuedRun = dispatcher.next()) {
if (claimedRuns.length >= availableSlots || isRunAdmissionClosed()) break;
const slots = { full: false };
let claimAttempts = 0;
for (let candidate = dispatcher.next(); candidate; candidate = dispatcher.next()) {
if (claimedRuns.length >= availableSlots || slots.full || isRunAdmissionClosed() || signal.aborted) break;
if (!retryRefused && !dispatcher.personRun(candidate) && claimBackedOff(candidate.id, new Date())) continue;
if (claimAttempts >= QUEUED_RUN_CLAIM_ATTEMPTS_PER_PASS) {
void startNextQueuedRunForAgent(agentId);
break;
}
claimAttempts += 1;
const [queuedRun] = await db.select().from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, candidate.id), eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.status, "queued")));
if (!queuedRun) continue;
let claimed: typeof heartbeatRuns.$inferSelect | null;
try {
claimed = await claimQueuedRun(queuedRun, companyAgents, deferredPostCommitEffects, deferredCancellations);
claimed = await claimQueuedRun(queuedRun, companyAgents, deferredPostCommitEffects, deferredCancellations, slots);
} catch (err) {
if (isPermanentClaimRejection(err)) {
rejectedClaims.push({ run: queuedRun, err });
@@ -2463,54 +2529,75 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
{ err, runId: queuedRun.id, agentId: queuedRun.agentId, companyId: queuedRun.companyId },
"queued heartbeat run claim was rejected; leaving it queued for the next recovery pass",
);
noteDeclinedAgedClaim(queuedRun, err.message);
noteDeclinedAgedClaim(candidate, err.message);
backOffClaim(candidate);
continue;
}
if (!claimed) {
noteDeclinedAgedClaim(queuedRun, "claim_declined");
if (slots.full) break;
noteDeclinedAgedClaim(candidate, "claim_declined");
backOffClaim(candidate);
continue;
}
claimedRuns.push(claimed);
dispatcher.claimed(queuedRun);
declinedAgedClaimAt.delete(queuedRun.id);
dispatcher.claimed(candidate);
declinedAgedClaimAt.delete(candidate.id);
claimBackoffs.delete(candidate.id);
// Start each run as soon as it is claimed. The claim committed a legacy
// controller lease that only the run's own execution renews; waiting
// for the rest of a slow batch to be claimed let that lease lapse.
startClaimedRun(claimed);
}
return claimedRuns;
}
function startClaimedRun(claimedRun: typeof heartbeatRuns.$inferSelect) {
const execution = executeRun(claimedRun.id).catch((err) => {
logger.error(
{ err, runId: claimedRun.id },
"queued heartbeat execution failed",
);
});
// Register the in-flight execution so drainActiveRunExecutions() can await
// it. executeRun resolves only after its finally block finishes flushing
// run rows/events, so awaiting this promise guarantees the run's writes
// have landed before a caller (e.g. a test's afterEach) mutates the DB.
activeRunExecutionPromises.add(execution);
void execution.finally(() => {
// drainActiveRunExecutions loops on activeRunExecutionPromises.size,
// so an entry that never clears here would hang it forever.
activeRunExecutionPromises.delete(execution);
});
}
}).finally(async () => {
// Dispatch promoted inputs after the agent start lock is released.
try {
await applyWakeQueuePostCommitEffects(deferredPostCommitEffects);
} finally {
if (cancellationReason) await cancelActiveForAgentInternal(agentId, cancellationReason);
await cancelRejectedQueuedRuns(rejectedClaims);
for (const cancel of deferredCancellations) {
await cancel().catch((err) => logger.warn({ err, agentId },
"queued heartbeat run refused by a pause hold could not be cancelled; the next claim retries it"));
}
}
});
function startClaimedRun(claimedRun: typeof heartbeatRuns.$inferSelect) {
const execution = outsideAgentDispatch(() => executeRun(claimedRun.id)).catch((err) => {
logger.error(
{ err, runId: claimedRun.id },
"queued heartbeat execution failed",
);
});
// Register the in-flight execution so drainActiveRunExecutions() can await
// it. executeRun resolves only after its finally block finishes flushing
// run rows/events, so awaiting this promise guarantees the run's writes
// have landed before a caller (e.g. a test's afterEach) mutates the DB.
activeRunExecutionPromises.add(execution);
void execution.finally(() => {
// drainActiveRunExecutions loops on activeRunExecutionPromises.size,
// so an entry that never clears here would hang it forever.
activeRunExecutionPromises.delete(execution);
});
}
}
async function listQueuedRunCandidates(agentId: string, cutoff: Date | null): Promise<QueuedRunCandidate[]> {
const contextText = (key: string) => sql<string | null>`case when jsonb_typeof(${heartbeatRuns.contextSnapshot}->${key}) = 'string'
then ${heartbeatRuns.contextSnapshot}->>${key} end`;
const rows = await db.select({
id: heartbeatRuns.id,
companyId: heartbeatRuns.companyId,
agentId: heartbeatRuns.agentId,
status: heartbeatRuns.status,
invocationSource: heartbeatRuns.invocationSource,
createdAt: heartbeatRuns.createdAt,
startedAt: heartbeatRuns.startedAt,
wakeupRequestId: heartbeatRuns.wakeupRequestId,
retryOfRunId: heartbeatRuns.retryOfRunId,
scheduledRetryAt: heartbeatRuns.scheduledRetryAt,
scheduledRetryAttempt: heartbeatRuns.scheduledRetryAttempt,
issueId: contextText("issueId"),
wakeReason: contextText("wakeReason"),
previousRunId: contextText("previousRunId"),
}).from(heartbeatRuns).where(and(
eq(heartbeatRuns.agentId, agentId),
eq(heartbeatRuns.status, "queued"),
cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined,
)).orderBy(asc(heartbeatRuns.createdAt));
return rows.map(({ issueId, wakeReason, previousRunId, ...run }) => ({
...run,
contextSnapshot: Object.fromEntries(Object.entries({ issueId, wakeReason, previousRunId }).filter(([, value]) => value !== null)),
}));
}
// Public wakeup entry point. Callers dispatch it fire-and-forget, so register
@@ -0,0 +1,21 @@
import { agents, heartbeatRuns, type Db } from "@paperclipai/db";
import { and, eq, sql } from "drizzle-orm";
import { maxConcurrentRunsSql } from "./timer-policy.js";
export class AgentRunSlotsFull extends Error {
constructor(readonly agentId: string, readonly running: number, readonly limit: number) {
super("The agent already runs as many runs at once as it allows.");
this.name = "AgentRunSlotsFull";
}
}
export async function assertAgentRunSlotFree(tx: Pick<Db, "select">, companyId: string, agentId: string) {
const [slots] = await tx.select({
limit: maxConcurrentRunsSql(),
running: sql<number>`(select count(*)::int from ${heartbeatRuns} where ${heartbeatRuns.companyId} = ${companyId}::uuid and ${heartbeatRuns.agentId} = ${agentId}::uuid and ${heartbeatRuns.status} = 'running')`,
}).from(agents).where(and(eq(agents.id, agentId), eq(agents.companyId, companyId)));
if (!slots) return;
const running = Number(slots.running);
const limit = Number(slots.limit);
if (running >= limit) throw new AgentRunSlotsFull(agentId, running, limit);
}
@@ -3,6 +3,7 @@ import { withExternalAdmissionGuard, assertNoExternalOverlap } from "../../modul
import { agentWakeupRequests, heartbeatRuns, issues, type Db } from "@paperclipai/db";
import { and, eq } from "drizzle-orm";
import type { PgUpdateSetSource } from "drizzle-orm/pg-core";
import { assertAgentRunSlotFree } from "../heartbeat/run-slot-admission.js";
import { claimNativeReviewExecutionLock, readNativeReviewAssignmentContext } from "./native-review-participant.js";
/** The run, wake, and issue lock form one reviewer admission claim. */
@@ -35,7 +36,10 @@ export async function claimQueuedNativeReviewRun(db: Db, input: {
)).for("update");
if (!run || run.status !== "queued" || run.wakeupRequestId !== input.run.wakeupRequestId) return null;
if ((run.contextSnapshot as Record<string, unknown> | null)?.issueId !== issueId) return null;
await withExternalAdmissionGuard(tx, run.companyId, run.agentId, () => assertNoExternalOverlap(tx, run.companyId, run.agentId));
await withExternalAdmissionGuard(tx, run.companyId, run.agentId, async () => {
await assertNoExternalOverlap(tx, run.companyId, run.agentId);
await assertAgentRunSlotFree(tx, run.companyId, run.agentId);
});
const locked = await claimNativeReviewExecutionLock(tx as unknown as Db, {
companyId: run.companyId, issueId, agentId: run.agentId, runId: run.id,
contextSnapshot: context, agentNameKey: input.agentNameKey, claimedAt: input.claimedAt,
@@ -48,7 +48,7 @@ describe("automatic retained workspace recovery", () => {
const experimental = await instanceSettingsService(db).getExperimental();
let stoppedWorker: Awaited<ReturnType<typeof stoppedWorkspaceWorker>> | undefined;
let storage: { mockRestore(): void } | undefined;
const starts = vi.spyOn(agentStarts, "withAgentStartLock");
const starts = vi.spyOn(agentStarts, "dispatchForAgent");
const retryResumes: Promise<void>[] = [];
let resumeSpy: { mockRestore(): void } | undefined;
let ownedHeartbeat: ReturnType<typeof heartbeatService> | undefined;
@@ -178,7 +178,8 @@ describe("automatic retained workspace recovery", () => {
});
const app = boardRecoveryApp(undefined, heartbeat);
const gate = new Promise<void>(resolve => { releaseStart = resolve; });
heldStart = agentStarts.withAgentStartLock(source.agentId, () => gate);
heldStart = agentStarts.dispatchForAgent(source.agentId, { request: null, idle: undefined, merge: () => null,
run: async () => ({ value: await gate }) });
const receipt = mode === "automatic_lost" ? null : (await request(app).post(`/api/issues/${f.issueId}/retry`)
.send({ observedRunId: f.runId, idempotencyKey: randomUUID() }).expect(202)).body;
if (mode === "automatic_lost") {