diff --git a/server/src/__tests__/heartbeat-start-lock.test.ts b/server/src/__tests__/heartbeat-start-lock.test.ts deleted file mode 100644 index 29aefdfb6da..00000000000 --- a/server/src/__tests__/heartbeat-start-lock.test.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { randomUUID } from "node:crypto"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import { withAgentStartLock } from "../services/agent-start-lock.ts"; - -describe("heartbeat agent start lock", () => { - afterEach(() => { - vi.useRealTimers(); - }); - - it("does not let a stale start lock freeze later queued-run starts", async () => { - vi.useFakeTimers(); - - const agentId = randomUUID(); - const firstStart = vi.fn(() => new Promise(() => undefined)); - const secondStart = vi.fn(async () => "started"); - - void withAgentStartLock(agentId, firstStart); - await Promise.resolve(); - expect(firstStart).toHaveBeenCalledTimes(1); - - const secondStartResult = withAgentStartLock(agentId, secondStart); - await Promise.resolve(); - expect(secondStart).not.toHaveBeenCalled(); - - await vi.advanceTimersByTimeAsync(30_000); - - await expect(secondStartResult).resolves.toBe("started"); - expect(secondStart).toHaveBeenCalledTimes(1); - }); -}); diff --git a/server/src/__tests__/queued-run-dispatch-stacking.test.ts b/server/src/__tests__/queued-run-dispatch-stacking.test.ts new file mode 100644 index 00000000000..ca78f15241f --- /dev/null +++ b/server/src/__tests__/queued-run-dispatch-stacking.test.ts @@ -0,0 +1,237 @@ +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + agentWakeupRequests, agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issues, +} from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { heartbeatService } from "../services/heartbeat.ts"; +import { runningProcesses } from "../adapters/index.ts"; +import { logger } from "../middleware/logger.js"; +import { isAgentDispatchActive } from "../services/agent-start-lock.js"; + +const held = vi.hoisted(() => { + const state = { + release: () => undefined as void, + released: Promise.resolve(), + reset() { + state.released = new Promise((resolve) => { state.release = resolve; }); + }, + }; + return state; +}) + +const passes = vi.hoisted(() => ({ started: 0, active: 0, peak: 0 })); + +vi.mock("../adapters/index.ts", async () => { + const actual = await vi.importActual("../adapters/index.ts"); + return { + ...actual, + getServerAdapter: vi.fn(() => ({ + supportsLocalAgentJwt: false, + execute: async () => { + await held.released; + return { exitCode: 0, signal: null, timedOut: false, errorMessage: null, summary: "done", provider: "test", model: "test" }; + }, + })), + }; +}); + +vi.mock("../services/agent-start-lock.js", async () => { + const actual = await vi.importActual("../services/agent-start-lock.js"); + const measure = async (pass: () => Promise) => { + passes.started += 1; + passes.active += 1; + passes.peak = Math.max(passes.peak, passes.active); + try { + return await pass(); + } finally { + passes.active -= 1; + } + }; + return { + ...actual, + dispatchForAgent: (agentId: string, spec: Parameters[1]) => + actual.dispatchForAgent(agentId, { ...spec, run: (request, signal) => measure(() => spec.run(request, signal)) }), + }; +}); + +const support = await getEmbeddedPostgresTestSupport(); +const suite = support.supported ? describe : describe.skip; + +const CAP = 4; +const UNCLAIMABLE = 12; +const CLAIMABLE = 3 * CAP; +const TRIGGERS = 40; + +suite("queued-run dispatch under a trigger burst", () => { + let db!: ReturnType; + let temporary: Awaited> | null = null; + let companyId = "", userId = ""; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-queued-dispatch-stacking-"); + db = createDb(temporary.connectionString); + await db.execute(sql`create table running_peaks (agent_id uuid not null, running int not null)`); + await db.execute(sql`create function record_running_peak() returns trigger language plpgsql as $$ + begin + insert into running_peaks select new.agent_id, count(*) from heartbeat_runs where agent_id = new.agent_id and status = 'running'; + return new; + end $$`); + await db.execute(sql`create trigger record_running_peak after update of status on heartbeat_runs + for each row when (new.status = 'running') execute function record_running_peak()`); + }, 60_000); + + afterEach(async () => { + vi.restoreAllMocks(); + held.release(); + runningProcesses.clear(); + await db.execute(sql`truncate table ${companies} cascade`); + }); + + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }); + + async function seedAgent() { + userId = randomUUID(); + companyId = randomUUID(); + const agentId = randomUUID(); + await db.insert(authUsers).values({ id: userId, name: "Board", email: `${userId}@example.test`, + emailVerified: true, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companies).values({ id: companyId, name: "Burst", issuePrefix: `B${companyId.replaceAll("-", "").slice(0, 6)}`, + requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: userId }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, status: "active", membershipRole: "owner" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Codie", role: "engineer", status: "idle", + adapterType: "codex_local", adapterConfig: {}, + runtimeConfig: { heartbeat: { enabled: true, wakeOnDemand: true, maxConcurrentRuns: CAP } }, permissions: {} }); + return agentId; + } + + async function task(agentId: string) { + const [row] = await db.insert(issues).values({ companyId, title: "Continue the work", status: "in_progress", assigneeAgentId: agentId }).returning(); + return row!; + } + + async function finishedRun(agentId: string, issueId: string, minutesAgo: number) { + const finishedAt = new Date(Date.now() - minutesAgo * 60_000); + const [row] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId, invocationSource: "assignment", + status: "failed", errorCode: "server_shutdown_interrupted", startedAt: new Date(finishedAt.getTime() - 60_000), finishedAt, + createdAt: new Date(finishedAt.getTime() - 120_000), contextSnapshot: { issueId, wakeReason: "issue_assigned" } }).returning(); + return row!; + } + + async function queuedContinuation(agentId: string, input: { minutesAgo: number; wakeReason: string; sourceIssueId?: string }) { + const issue = await task(agentId); + const source = await finishedRun(agentId, input.sourceIssueId ?? issue.id, input.minutesAgo + 5); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "automation", triggerDetail: "system", + reason: input.wakeReason, status: "queued", requestedByActorType: "system", requestedByActorId: null, + payload: { issueId: issue.id } }).returning(); + const [row] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: issue.id, invocationSource: "automation", + triggerDetail: "system", status: "queued", retryOfRunId: source.id, wakeupRequestId: wake!.id, + createdAt: new Date(Date.now() - input.minutesAgo * 60_000), + contextSnapshot: { issueId: issue.id, taskId: issue.id, wakeReason: input.wakeReason, previousRunId: source.id, + source: "issue.assigned_work_invariant", paperclipWorkspace: { notes: "x".repeat(16_000) } } }).returning(); + await db.update(agentWakeupRequests).set({ runId: row!.id }).where(eq(agentWakeupRequests.id, wake!.id)); + return row!; + } + + const peakRunning = async (agentId: string) => Number((await db.execute(sql`select coalesce(max(running), 0) as peak + from running_peaks where agent_id = ${agentId}::uuid`))[0]!.peak); + + it("runs one pass at a time for a burst of triggers, offers a refused run once, and never runs past the agent's limit", async () => { + held.reset(); + passes.started = 0; passes.active = 0; passes.peak = 0; + const agentId = await seedAgent(); + const otherTask = await task(agentId); + const unclaimable = new Set(); + for (let index = 0; index < UNCLAIMABLE; index += 1) { + unclaimable.add((await queuedContinuation(agentId, { minutesAgo: 180 - index, wakeReason: "issue_continuation_needed", + sourceIssueId: otherTask.id })).id); + } + for (let index = 0; index < CLAIMABLE; index += 1) { + await queuedContinuation(agentId, { minutesAgo: 30 - index, wakeReason: "issue_assigned" }); + } + + const realNow = Date.now.bind(Date); + let skewMs = 0; + vi.spyOn(Date, "now").mockImplementation(() => realNow() + skewMs); + const offers = new Map(); + let inFlightClaims = 0; + let peakInFlightClaims = 0; + const triggers: Array> = []; + let burstFired = false; + const heartbeat = heartbeatService(db, { + beforeChatControlRecoveryCheck: async ({ runId, stage }) => { + if (stage !== "claim") return; + offers.set(runId, (offers.get(runId) ?? 0) + 1); + inFlightClaims += 1; + peakInFlightClaims = Math.max(peakInFlightClaims, inFlightClaims); + try { + if (!burstFired) { + burstFired = true; + for (let index = 0; index < TRIGGERS; index += 1) { + skewMs += 31_000; + triggers.push(heartbeat.resumeQueuedRunsForAgent(companyId, agentId)); + await new Promise((resolve) => setTimeout(resolve, 100)); + } + skewMs = 0; + } + await new Promise((resolve) => setTimeout(resolve, 25)); + } finally { + inFlightClaims -= 1; + } + }, + }); + + try { + await heartbeat.resumeQueuedRunsForAgent(companyId, agentId); + await Promise.all(triggers); + const refusedOffers = [...unclaimable].reduce((sum, id) => sum + (offers.get(id) ?? 0), 0); + expect({ + passes: passes.started, + peakConcurrentPasses: passes.peak, + peakInFlightClaims, + refusedOffersPerTrigger: refusedOffers / (TRIGGERS + 1), + peakRunning: await peakRunning(agentId), + }).toEqual({ + passes: 2, + peakConcurrentPasses: 1, + peakInFlightClaims: 1, + refusedOffersPerTrigger: UNCLAIMABLE / (TRIGGERS + 1), + peakRunning: CAP, + }); + await vi.waitFor(() => { + expect(passes.active).toBe(0); + expect(isAgentDispatchActive(agentId)).toBe(false); + }, { timeout: 10_000 }); + + } finally { + held.release(); + await heartbeat.drainActiveRunExecutions(); + } + }, 120_000); + + it.each(["declined", "rejected"] as const)("waits a bounded time before offering a %s run again, and offers it again once that time passes", async (refusal) => { + const agentId = await seedAgent(); + if (refusal === "rejected") { + await db.update(companies).set({ defaultResponsibleUserId: null }).where(eq(companies.id, companyId)); + await db.delete(companyMemberships).where(eq(companyMemberships.companyId, companyId)); + } + const otherTask = await task(agentId); + const refused = await queuedContinuation(agentId, { minutesAgo: 180, wakeReason: "issue_continuation_needed", sourceIssueId: otherTask.id }); + const warn = vi.spyOn(logger, "warn"); + const offersOf = () => warn.mock.calls.filter(([fields, message]) => typeof message === "string" + && message.startsWith("long-queued heartbeat run was offered a free slot") + && (fields as { runId?: string }).runId === refused.id).length; + const heartbeat = heartbeatService(db); + try { + for (let index = 0; index < 10; index += 1) await heartbeat.resumeQueuedRunsForAgent(companyId, agentId); + expect(offersOf()).toBe(1); + await vi.waitFor(async () => { + await heartbeat.resumeQueuedRunsForAgent(companyId, agentId); + expect(offersOf()).toBe(2); + }, { timeout: 20_000, interval: 1_000 }); + } finally { + await heartbeat.drainActiveRunExecutions(); + } + }, 60_000); +}); diff --git a/server/src/services/agent-start-lock.test.ts b/server/src/services/agent-start-lock.test.ts new file mode 100644 index 00000000000..dee3471a2d5 --- /dev/null +++ b/server/src/services/agent-start-lock.test.ts @@ -0,0 +1,140 @@ +import { randomUUID } from "node:crypto"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + AGENT_DISPATCH_PASS_DEADLINE_MS, + dispatchForAgent, + isAgentDispatchActive, + type AgentDispatchPass, +} from "./agent-start-lock.js"; + +function gate() { + let open!: () => void; + const opened = new Promise((resolve) => { open = resolve; }); + return { opened, open }; +} + +function meter() { + const stats = { passes: 0, active: 0, peak: 0, requests: [] as boolean[] }; + const spec = (attach: boolean, body: (signal: AbortSignal) => Promise): AgentDispatchPass => ({ + request: attach, + idle: -1, + merge: (current, next) => current || next, + async run(request, signal) { + stats.passes += 1; + stats.active += 1; + stats.peak = Math.max(stats.peak, stats.active); + stats.requests.push(request); + const pass = stats.passes; + try { + await body(signal); + } finally { + stats.active -= 1; + } + return { value: pass }; + }, + }); + return { stats, spec }; +} + +describe("agent dispatch passes", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("runs one pass per agent and folds every trigger that arrives during it into one follow-up pass", async () => { + const agentId = randomUUID(); + const { stats, spec } = meter(); + const first = gate(); + const firstPass = dispatchForAgent(agentId, spec(false, () => first.opened)); + const burst = Array.from({ length: 50 }, (_, index) => + dispatchForAgent(agentId, spec(index === 7, async () => undefined))); + expect(stats).toMatchObject({ passes: 1, peak: 1 }); + first.open(); + expect(await firstPass).toBe(1); + expect(new Set(await Promise.all(burst))).toEqual(new Set([2])); + expect(stats).toMatchObject({ passes: 2, peak: 1, requests: [false, true] }); + expect(isAgentDispatchActive(agentId)).toBe(false); + }); + + it("keeps a second pass out while a slow pass runs past the old 30 second stale limit", async () => { + vi.useFakeTimers(); + const agentId = randomUUID(); + const { stats, spec } = meter(); + const slow = gate(); + const firstPass = dispatchForAgent(agentId, spec(false, () => slow.opened)); + const triggers: Array> = []; + for (let second = 0; second < 45; second += 1) { + await vi.advanceTimersByTimeAsync(1_000); + triggers.push(dispatchForAgent(agentId, spec(false, async () => undefined))); + } + expect(stats).toMatchObject({ passes: 1, peak: 1 }); + slow.open(); + await firstPass; + await Promise.all(triggers); + expect(stats).toMatchObject({ passes: 2, peak: 1 }); + }); + + it("stops a hung pass at its deadline and lets the waiting trigger run without overlapping it", async () => { + vi.useFakeTimers(); + const agentId = randomUUID(); + const { stats, spec } = meter(); + let hungSignal: AbortSignal | null = null; + void dispatchForAgent(agentId, spec(false, (signal) => { + hungSignal = signal; + return new Promise(() => undefined); + })); + const waiting = dispatchForAgent(agentId, spec(false, async () => undefined)); + await vi.advanceTimersByTimeAsync(AGENT_DISPATCH_PASS_DEADLINE_MS - 1); + expect(stats.passes).toBe(1); + expect(hungSignal!.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(1); + expect(hungSignal!.aborted).toBe(true); + expect(await waiting).toBe(2); + expect(stats.passes).toBe(2); + expect(isAgentDispatchActive(agentId)).toBe(false); + expect(await dispatchForAgent(agentId, spec(false, async () => undefined))).toBe(3); + }); + + it("does not wait on itself when a pass triggers its own agent", async () => { + const agentId = randomUUID(); + const { stats, spec } = meter(); + let nested: Promise | null = null; + const pass = dispatchForAgent(agentId, spec(false, async () => { + nested = dispatchForAgent(agentId, spec(false, async () => undefined)); + expect(await nested).toBe(-1); + })); + expect(await pass).toBe(1); + await vi.waitFor(() => expect(stats.passes).toBe(2)); + expect(stats.peak).toBe(1); + }); + + it("settles a pass's effects after the next pass may start", async () => { + const agentId = randomUUID(); + const order: string[] = []; + const release = gate(); + const pass = dispatchForAgent(agentId, { + request: null, idle: null, merge: () => null, + async run() { + await release.opened; + return { + value: null, + settle: async () => { + order.push(`settle:${isAgentDispatchActive(agentId)}`); + await dispatchForAgent(agentId, { request: null, idle: null, merge: () => null, + async run() { order.push("nested"); return { value: null }; } }); + }, + }; + }, + }); + release.open(); + await pass; + expect(order).toEqual(["settle:false", "nested"]); + }); + + it("passes a failed pass's error to its caller and frees the agent", async () => { + const agentId = randomUUID(); + const { spec } = meter(); + await expect(dispatchForAgent(agentId, spec(false, async () => { throw new Error("boom"); }))).rejects.toThrow("boom"); + expect(isAgentDispatchActive(agentId)).toBe(false); + }); +}); diff --git a/server/src/services/agent-start-lock.ts b/server/src/services/agent-start-lock.ts index 38cbaf74c6e..9b4aa4abcd0 100644 --- a/server/src/services/agent-start-lock.ts +++ b/server/src/services/agent-start-lock.ts @@ -1,48 +1,90 @@ +import { AsyncLocalStorage } from "node:async_hooks"; import { logger } from "../middleware/logger.js"; -const AGENT_START_LOCK_STALE_MS = 30_000; -const startLocksByAgent = new Map; startedAtMs: number }>(); +export const AGENT_DISPATCH_PASS_DEADLINE_MS = 60_000; -async function waitForAgentStartLock(agentId: string, lock: { promise: Promise; startedAtMs: number }) { - const elapsedMs = Date.now() - lock.startedAtMs; - const remainingMs = AGENT_START_LOCK_STALE_MS - elapsedMs; - if (remainingMs <= 0) { - logger.warn({ agentId, staleMs: elapsedMs }, "agent start lock stale; continuing queued-run start"); - return; - } +export type AgentDispatchPass = { + request: R; + idle: T; + merge(current: R, next: R): R; + run(request: R, signal: AbortSignal): Promise<{ value: T; settle?: () => Promise }>; + track?(pass: Promise): void; +}; - let timedOut = false; - let timeout: ReturnType | null = null; - await Promise.race([ - lock.promise, - new Promise((resolve) => { - timeout = setTimeout(() => { - timedOut = true; - resolve(); - }, remainingMs); - }), - ]); - if (timeout) clearTimeout(timeout); +type Deferred = { promise: Promise; resolve(value: T): void; reject(err: unknown): void }; +type FollowUp = { request: unknown; spec: AgentDispatchPass; done: Deferred }; +type AgentDispatchState = { followUp: FollowUp | null }; - if (timedOut) { - logger.warn({ agentId, staleMs: AGENT_START_LOCK_STALE_MS }, "agent start lock timed out; continuing queued-run start"); - } +const dispatchStates = new Map(); +const dispatchingAgent = new AsyncLocalStorage(); + +function deferred(): Deferred { + let resolve!: (value: T) => void; + let reject!: (err: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + promise.catch(() => undefined); + return { promise, resolve, reject }; } -export async function withAgentStartLock(agentId: string, fn: () => Promise) { - const previous = startLocksByAgent.get(agentId); - const waitForPrevious = previous ? waitForAgentStartLock(agentId, previous) : Promise.resolve(); - const run = waitForPrevious.then(fn); - const marker = run.then( - () => undefined, - () => undefined, +export function isAgentDispatchActive(agentId: string) { + return dispatchStates.has(agentId); +} + +export function outsideAgentDispatch(fn: () => T): T { + return dispatchingAgent.exit(fn); +} + +export function dispatchForAgent(agentId: string, spec: AgentDispatchPass): Promise { + const state = dispatchStates.get(agentId); + if (!state) return startPass(agentId, spec, spec.request); + if (state.followUp) { + state.followUp.request = spec.merge(state.followUp.request as R, spec.request); + state.followUp.spec = spec as AgentDispatchPass; + } else { + state.followUp = { request: spec.request, spec: spec as AgentDispatchPass, done: deferred() }; + } + if (dispatchingAgent.getStore() === agentId) return Promise.resolve(spec.idle); + return state.followUp.done.promise as Promise; +} + +function startPass(agentId: string, spec: AgentDispatchPass, request: R): Promise { + const state: AgentDispatchState = { followUp: null }; + dispatchStates.set(agentId, state); + const controller = new AbortController(); + const expired = deferred(); + let released = false; + const release = () => { + if (released) return; + released = true; + if (dispatchStates.get(agentId) === state) dispatchStates.delete(agentId); + const next = state.followUp; + state.followUp = null; + if (next) startPass(agentId, next.spec, next.request).then(next.done.resolve, next.done.reject); + }; + const deadline = setTimeout(() => { + logger.warn({ agentId, deadlineMs: AGENT_DISPATCH_PASS_DEADLINE_MS }, + "queued-run dispatch pass passed its deadline; it stops claiming and the next pass may start"); + controller.abort(); + release(); + expired.resolve(); + }, AGENT_DISPATCH_PASS_DEADLINE_MS); + deadline.unref?.(); + const pass = dispatchingAgent.run(agentId, () => spec.run(request, controller.signal)).then( + async ({ value, settle }) => { + clearTimeout(deadline); + release(); + if (settle) await outsideAgentDispatch(settle); + return value; + }, + (err: unknown) => { + clearTimeout(deadline); + release(); + throw err; + }, ); - startLocksByAgent.set(agentId, { promise: marker, startedAtMs: Date.now() }); - try { - return await run; - } finally { - if (startLocksByAgent.get(agentId)?.promise === marker) { - startLocksByAgent.delete(agentId); - } - } + spec.track?.(Promise.race([pass.then(() => undefined, () => undefined), expired.promise])); + return pass; } diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index d3f1d08b284..2ea2c60154a 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -12,7 +12,7 @@ import { hasAutomaticInteractionDeliveryHold, INTERACTION_DELIVERY_DENIED, reche import { deferredWorkIntentWait } from "../work-intent-outbox.js"; import { notifyIssueRecovery } from "../issue-recovery-notifications.js"; import { createHeartbeatTimerPolicy } from "./timer-policy.js"; -import { QUEUED_RUN_AGING_MS, createQueuedRunDispatcher, hasQueuedRunAged } from "./queued-run-dispatch-order.js"; +import { QUEUED_RUN_AGING_MS, createQueuedRunDispatcher, hasQueuedRunAged, queuedRunDispatchClass } from "./queued-run-dispatch-order.js"; import { queuedRunDispatchFacts } from "./queued-run-line.js"; import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { PersistedExecutionWaitRefusal, writeDeferredExecutionWait } from "../deferred-execution-wait.js"; @@ -188,7 +188,8 @@ import { } from "../execution-workspace-policy.js"; import { readContinuationAttempt } from "../recovery/index.js"; import { allowsDependencyBlockedWake, extractWakeCommentIds, wakeChatConversationsConflict } from "../../modules/run-dispatch/index.js"; -import { withAgentStartLock } from "../agent-start-lock.js"; +import { dispatchForAgent, outsideAgentDispatch } from "../agent-start-lock.js"; +import { AgentRunSlotsFull, assertAgentRunSlotFree } from "./run-slot-admission.js"; import { evaluateAgentInvokability, shouldCancelRunsForNonInvokableAgent, @@ -214,6 +215,9 @@ import { hasRetainedExternalChatRetry, type PostCommitEffect as WakeQueuePostCom import { crashPoint } from "@paperclipai/shared/crash-points"; type HeartbeatRun = typeof heartbeatRuns.$inferSelect; +type QueuedRunCandidate = Pick; +type QueuedDispatchRequest = { attachSetupRequest: boolean; retryRefused: boolean }; type HeartbeatRunState = ReturnType; type RunPreparation = ReturnType; @@ -375,6 +379,9 @@ export function shouldQueueFollowupForRunningIssueWake(input: { /** Agent Chat outbox receipts. The outbox, not a skipped receipt, owns a capped message. */ const CONVERSATION_COMMENT_KEY_PREFIX = "conversation-comment:"; +const QUEUED_RUN_CLAIM_ATTEMPTS_PER_PASS = 24; +const QUEUED_RUN_CLAIM_BACKOFF_BASE_MS = 5_000; +const QUEUED_RUN_CLAIM_BACKOFF_MAX_MS = 60_000; function dailyLimitConversationRefusal(cap: { reason: string; observed: number; limit: number }) { // No receipt is written, so the person's message stays owed in the outbox // until the limit resets (agent-conversations.ts conversationDeliveryHold). @@ -597,7 +604,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende async function listQueuedRunDependencyReadiness( companyId: string, - queuedRuns: Array, + queuedRuns: Array>, ) { const issueIds = [ ...new Set( @@ -960,6 +967,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende companyAgents?: AgentOrgRow[], deferredPostCommitEffects?: WakeQueuePostCommitEffect[], deferredCancellations?: Array<() => Promise>, + slots?: { full: boolean }, ) { if (run.status !== "queued") return run; // Every start path ends here (dispatch passes, executeRun of a queued run, @@ -1583,7 +1591,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende // envelope. Preserve their established claim path; only an // explicitly bound queued-message envelope is subject to the // live-comment discard gate below. - await withExternalAdmissionGuard(tx, run.companyId, run.agentId, () => assertNoExternalOverlap(tx, run.companyId, run.agentId)); + await withExternalAdmissionGuard(tx, run.companyId, run.agentId, async () => { + await assertNoExternalOverlap(tx, run.companyId, run.agentId); + await assertAgentRunSlotFree(tx, run.companyId, run.agentId); + }); await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); const [claimedRun] = await tx .update(heartbeatRuns) @@ -1679,7 +1690,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende }; } - await withExternalAdmissionGuard(tx, run.companyId, run.agentId, () => assertNoExternalOverlap(tx, run.companyId, run.agentId)); + await withExternalAdmissionGuard(tx, run.companyId, run.agentId, async () => { + await assertNoExternalOverlap(tx, run.companyId, run.agentId); + await assertAgentRunSlotFree(tx, run.companyId, run.agentId); + }); await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); await tx .update(agentWakeupRequests) @@ -1723,6 +1737,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende .catch((error) => { if (isExternalChatWaitAuthorizationContention(error) || error instanceof RunAdmissionClosedDuringClaim) return { kind: "stale" as const, run: null }; + if (error instanceof AgentRunSlotsFull) { + if (slots) slots.full = true; + return { kind: "stale" as const, run: null }; + } throw error; }) : null; @@ -1795,7 +1813,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.agentId, run.agentId), )).for("update"); if (!lockedRun || lockedRun.status !== "queued") return null; - await withExternalAdmissionGuard(claimTx, run.companyId, run.agentId, () => assertNoExternalOverlap(claimTx, run.companyId, run.agentId)); + await withExternalAdmissionGuard(claimTx, run.companyId, run.agentId, async () => { + await assertNoExternalOverlap(claimTx, run.companyId, run.agentId); + await assertAgentRunSlotFree(claimTx, run.companyId, run.agentId); + }); await notifyChatRunMilestones(claimTx, { companyId: run.companyId, runId: run.id, issueId: issueId ?? undefined }); const claimedRun = await claimTx.update(heartbeatRuns).set(claimValues).where(and( eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "queued"), @@ -1806,6 +1827,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende }); }).catch((error: unknown) => { if (error instanceof RunAdmissionClosedDuringClaim) return null; + if (error instanceof AgentRunSlotsFull) { + if (slots) slots.full = true; + return null; + } throw error; }); if (!claimed) return null; @@ -2252,7 +2277,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende const agentIds = [...new Set(queuedRuns.map((r) => r.agentId))]; for (const agentId of agentIds) { - await startNextQueuedRunForAgent(agentId, true); + await startNextQueuedRunForAgent(agentId, true, true); } } @@ -2336,7 +2361,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende const declinedAgedClaimAt = new Map(); - function noteDeclinedAgedClaim(run: typeof heartbeatRuns.$inferSelect, reason: string) { + function noteDeclinedAgedClaim(run: QueuedRunCandidate, reason: string) { const now = Date.now(); if (!hasQueuedRunAged(run, new Date(now))) return; declinedAgedClaimAt.set(run.id, now); @@ -2345,6 +2370,25 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende "long-queued heartbeat run was offered a free slot but could not be claimed; it stays queued and the next oldest run takes its turn"); } + const claimBackoffs = new Map(); + + function claimBackedOff(runId: string, now: Date) { + return (claimBackoffs.get(runId)?.until ?? 0) > now.getTime(); + } + + function backOffClaim(run: QueuedRunCandidate) { + const attempts = (claimBackoffs.get(run.id)?.attempts ?? 0) + 1; + const delayMs = Math.min(QUEUED_RUN_CLAIM_BACKOFF_BASE_MS * 2 ** (attempts - 1), QUEUED_RUN_CLAIM_BACKOFF_MAX_MS); + claimBackoffs.set(run.id, { agentId: run.agentId, attempts, until: Date.now() + delayMs }); + } + + function forgetSettledClaimBackoffs(agentId: string, queuedRuns: QueuedRunCandidate[]) { + const queued = new Set(queuedRuns.map(run => run.id)); + for (const [runId, backoff] of claimBackoffs) { + if (backoff.agentId === agentId && !queued.has(runId)) claimBackoffs.delete(runId); + } + } + function pruneDeclinedAgedClaims(now: Date) { for (const [id, at] of declinedAgedClaimAt) { if (now.getTime() - at >= QUEUED_RUN_AGING_MS) declinedAgedClaimAt.delete(id); @@ -2353,36 +2397,78 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende async function queuedRunDispatcherForAgent( agent: typeof agents.$inferSelect, - queuedRuns: Array, + queuedRuns: QueuedRunCandidate[], dependencyReadiness: Map, issueById: Map, ) { const now = new Date(); pruneDeclinedAgedClaims(now); const { facts, dispatchInput } = await queuedRunDispatchFacts(db, { - agent, queuedRuns, dependencyReadiness, issueById, now, shareDeferred: runId => declinedAgedClaimAt.has(runId) }); + agent, queuedRuns, dependencyReadiness, issueById, now, + shareDeferred: runId => declinedAgedClaimAt.has(runId) || claimBackedOff(runId, now) }); const dispatcher = createQueuedRunDispatcher(facts, dispatchInput); const byRun = new Map(facts.map(entry => [entry.run, entry])); return { next: () => dispatcher.next()?.run ?? null, - claimed: (run: typeof heartbeatRuns.$inferSelect) => { const entry = byRun.get(run); if (entry) dispatcher.claimed(entry); }, + claimed: (run: QueuedRunCandidate) => { const entry = byRun.get(run); if (entry) dispatcher.claimed(entry); }, + personRun: (run: QueuedRunCandidate) => { + const entry = byRun.get(run); + return Boolean(entry && queuedRunDispatchClass(entry, now) === "person"); + }, + personOrigin: (run: QueuedRunCandidate) => byRun.get(run)?.requestedByUser ?? false, }; } - async function startNextQueuedRunForAgent(agentId: string, attachSetupRequest = false) { - // Null unless this server has not read the run-start hand-over recently. - const refreshing = refreshRunAdmission(); - if (refreshing) await refreshing; - if (isRunAdmissionClosed() || (await getSchedulingSuppression()).suppressed) return []; - const cutoff = await getWorktreeExecutionCutoff(); - // Cancelled after the start lock is released: cancelRunInternal promotes the - // agent's next queued run, which takes this same lock. + function startNextQueuedRunForAgent(agentId: string, attachSetupRequest = false, retryRefused = false) { + return dispatchForAgent>(agentId, { + request: { attachSetupRequest, retryRefused }, + idle: [], + merge: (current, next) => ({ + attachSetupRequest: current.attachSetupRequest || next.attachSetupRequest, + retryRefused: current.retryRefused || next.retryRefused, + }), + run: (request, signal) => runQueuedDispatchPass(agentId, request, signal), + track: (pass) => { + const tracked = pass.then(() => undefined); + activeRunExecutionPromises.add(tracked); + void tracked.finally(() => activeRunExecutionPromises.delete(tracked)); + }, + }); + } + + async function runQueuedDispatchPass(agentId: string, { attachSetupRequest, retryRefused }: QueuedDispatchRequest, signal: AbortSignal) { + // Cancelled after the pass ends: cancelRunInternal promotes the agent's + // next queued run, which asks for another pass of this agent. const rejectedClaims: Array<{ run: typeof heartbeatRuns.$inferSelect; err: HttpError }> = []; const deferredPostCommitEffects: WakeQueuePostCommitEffect[] = []; const deferredCancellations: Array<() => Promise> = []; let cancellationReason: string | undefined; + const settle = async () => { + // Dispatch promoted inputs after the pass has ended. + try { + await applyWakeQueuePostCommitEffects(deferredPostCommitEffects); + } finally { + if (cancellationReason) await cancelActiveForAgentInternal(agentId, cancellationReason); + await cancelRejectedQueuedRuns(rejectedClaims); + for (const cancel of deferredCancellations) { + await cancel().catch((err) => logger.warn({ err, agentId }, + "queued heartbeat run refused by a pause hold could not be cancelled; the next claim retries it")); + } + } + }; + try { + return { value: await claimQueuedRunsForAgent(), settle }; + } catch (err) { + await settle(); + throw err; + } - return withAgentStartLock(agentId, async () => { + async function claimQueuedRunsForAgent(): Promise> { + // Null unless this server has not read the run-start hand-over recently. + const refreshing = refreshRunAdmission(); + if (refreshing) await refreshing; + if (isRunAdmissionClosed() || (await getSchedulingSuppression()).suppressed) return []; + const cutoff = await getWorktreeExecutionCutoff(); const agent = await getAgent(agentId); if (!agent) return []; const invokability = await getAgentInvokability(agent); @@ -2395,64 +2481,44 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende } const policy = parseHeartbeatPolicy(agent); const runningCount = await countRunningRunsForAgent(agentId); - const availableSlots = Math.max( - 0, - policy.maxConcurrentRuns - runningCount, - ); + const availableSlots = Math.max(0, policy.maxConcurrentRuns - runningCount); if (availableSlots <= 0) return []; - const queuedRuns = await db - .select() - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.agentId, agentId), - eq(heartbeatRuns.status, "queued"), - cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined, - ), - ) - .orderBy(asc(heartbeatRuns.createdAt)); + const queuedRuns = await listQueuedRunCandidates(agentId, cutoff); + forgetSettledClaimBackoffs(agentId, queuedRuns); if (queuedRuns.length === 0) return []; if (queuedRuns.every(run => !run.startedAt) && !await canStartNewExternalBotWork(agent)) return []; - const dependencyReadiness = await listQueuedRunDependencyReadiness( - agent.companyId, - queuedRuns, - ); - const queuedIssueIds = [ - ...new Set( - queuedRuns - .map((run) => - readNonEmptyString(parseObject(run.contextSnapshot).issueId), - ) - .filter((issueId): issueId is string => Boolean(issueId)), - ), - ]; - const issueRows = await db - .select({ - id: issues.id, - status: issues.status, - priority: issues.priority, - }) - .from(issues) - .where( - queuedIssueIds.length > 0 - ? and( - eq(issues.companyId, agent.companyId), - inArray(issues.id, queuedIssueIds), - ) - : sql`false`, - ); + const dependencyReadiness = await listQueuedRunDependencyReadiness(agent.companyId, queuedRuns); + const queuedIssueIds = [...new Set(queuedRuns.flatMap((run) => { + const issueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); + return issueId ? [issueId] : []; + }))]; + const issueRows = queuedIssueIds.length > 0 + ? await db.select({ id: issues.id, status: issues.status, priority: issues.priority }).from(issues) + .where(and(eq(issues.companyId, agent.companyId), inArray(issues.id, queuedIssueIds))) + : []; const issueById = new Map(issueRows.map((row) => [row.id, row])); const companyAgents = await listCompanyAgentOrgRows(agent.companyId); const dispatcher = await queuedRunDispatcherForAgent(agent, queuedRuns, dependencyReadiness, issueById); const claimedRuns: Array = []; - for (let queuedRun = dispatcher.next(); queuedRun; queuedRun = dispatcher.next()) { - if (claimedRuns.length >= availableSlots || isRunAdmissionClosed()) break; + const slots = { full: false }; + let claimAttempts = 0; + for (let candidate = dispatcher.next(); candidate; candidate = dispatcher.next()) { + if (claimedRuns.length >= availableSlots || slots.full || isRunAdmissionClosed() || signal.aborted) break; + if (!retryRefused && !dispatcher.personRun(candidate) && claimBackedOff(candidate.id, new Date())) continue; + if (claimAttempts >= QUEUED_RUN_CLAIM_ATTEMPTS_PER_PASS) { + void startNextQueuedRunForAgent(agentId); + break; + } + claimAttempts += 1; + const [queuedRun] = await db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, candidate.id), eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.status, "queued"))); + if (!queuedRun) continue; let claimed: typeof heartbeatRuns.$inferSelect | null; try { - claimed = await claimQueuedRun(queuedRun, companyAgents, deferredPostCommitEffects, deferredCancellations); + claimed = await claimQueuedRun(queuedRun, companyAgents, deferredPostCommitEffects, deferredCancellations, slots); } catch (err) { if (isPermanentClaimRejection(err)) { rejectedClaims.push({ run: queuedRun, err }); @@ -2463,54 +2529,75 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende { err, runId: queuedRun.id, agentId: queuedRun.agentId, companyId: queuedRun.companyId }, "queued heartbeat run claim was rejected; leaving it queued for the next recovery pass", ); - noteDeclinedAgedClaim(queuedRun, err.message); + noteDeclinedAgedClaim(candidate, err.message); + backOffClaim(candidate); continue; } if (!claimed) { - noteDeclinedAgedClaim(queuedRun, "claim_declined"); + if (slots.full) break; + noteDeclinedAgedClaim(candidate, "claim_declined"); + backOffClaim(candidate); continue; } claimedRuns.push(claimed); - dispatcher.claimed(queuedRun); - declinedAgedClaimAt.delete(queuedRun.id); + dispatcher.claimed(candidate); + declinedAgedClaimAt.delete(candidate.id); + claimBackoffs.delete(candidate.id); // Start each run as soon as it is claimed. The claim committed a legacy // controller lease that only the run's own execution renews; waiting // for the rest of a slow batch to be claimed let that lease lapse. startClaimedRun(claimed); } return claimedRuns; + } - function startClaimedRun(claimedRun: typeof heartbeatRuns.$inferSelect) { - const execution = executeRun(claimedRun.id).catch((err) => { - logger.error( - { err, runId: claimedRun.id }, - "queued heartbeat execution failed", - ); - }); - // Register the in-flight execution so drainActiveRunExecutions() can await - // it. executeRun resolves only after its finally block finishes flushing - // run rows/events, so awaiting this promise guarantees the run's writes - // have landed before a caller (e.g. a test's afterEach) mutates the DB. - activeRunExecutionPromises.add(execution); - void execution.finally(() => { - // drainActiveRunExecutions loops on activeRunExecutionPromises.size, - // so an entry that never clears here would hang it forever. - activeRunExecutionPromises.delete(execution); - }); - } - }).finally(async () => { - // Dispatch promoted inputs after the agent start lock is released. - try { - await applyWakeQueuePostCommitEffects(deferredPostCommitEffects); - } finally { - if (cancellationReason) await cancelActiveForAgentInternal(agentId, cancellationReason); - await cancelRejectedQueuedRuns(rejectedClaims); - for (const cancel of deferredCancellations) { - await cancel().catch((err) => logger.warn({ err, agentId }, - "queued heartbeat run refused by a pause hold could not be cancelled; the next claim retries it")); - } - } - }); + function startClaimedRun(claimedRun: typeof heartbeatRuns.$inferSelect) { + const execution = outsideAgentDispatch(() => executeRun(claimedRun.id)).catch((err) => { + logger.error( + { err, runId: claimedRun.id }, + "queued heartbeat execution failed", + ); + }); + // Register the in-flight execution so drainActiveRunExecutions() can await + // it. executeRun resolves only after its finally block finishes flushing + // run rows/events, so awaiting this promise guarantees the run's writes + // have landed before a caller (e.g. a test's afterEach) mutates the DB. + activeRunExecutionPromises.add(execution); + void execution.finally(() => { + // drainActiveRunExecutions loops on activeRunExecutionPromises.size, + // so an entry that never clears here would hang it forever. + activeRunExecutionPromises.delete(execution); + }); + } + } + + async function listQueuedRunCandidates(agentId: string, cutoff: Date | null): Promise { + const contextText = (key: string) => sql`case when jsonb_typeof(${heartbeatRuns.contextSnapshot}->${key}) = 'string' + then ${heartbeatRuns.contextSnapshot}->>${key} end`; + const rows = await db.select({ + id: heartbeatRuns.id, + companyId: heartbeatRuns.companyId, + agentId: heartbeatRuns.agentId, + status: heartbeatRuns.status, + invocationSource: heartbeatRuns.invocationSource, + createdAt: heartbeatRuns.createdAt, + startedAt: heartbeatRuns.startedAt, + wakeupRequestId: heartbeatRuns.wakeupRequestId, + retryOfRunId: heartbeatRuns.retryOfRunId, + scheduledRetryAt: heartbeatRuns.scheduledRetryAt, + scheduledRetryAttempt: heartbeatRuns.scheduledRetryAttempt, + issueId: contextText("issueId"), + wakeReason: contextText("wakeReason"), + previousRunId: contextText("previousRunId"), + }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.agentId, agentId), + eq(heartbeatRuns.status, "queued"), + cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined, + )).orderBy(asc(heartbeatRuns.createdAt)); + return rows.map(({ issueId, wakeReason, previousRunId, ...run }) => ({ + ...run, + contextSnapshot: Object.fromEntries(Object.entries({ issueId, wakeReason, previousRunId }).filter(([, value]) => value !== null)), + })); } // Public wakeup entry point. Callers dispatch it fire-and-forget, so register diff --git a/server/src/services/heartbeat/run-slot-admission.ts b/server/src/services/heartbeat/run-slot-admission.ts new file mode 100644 index 00000000000..8034ffa0e12 --- /dev/null +++ b/server/src/services/heartbeat/run-slot-admission.ts @@ -0,0 +1,21 @@ +import { agents, heartbeatRuns, type Db } from "@paperclipai/db"; +import { and, eq, sql } from "drizzle-orm"; +import { maxConcurrentRunsSql } from "./timer-policy.js"; + +export class AgentRunSlotsFull extends Error { + constructor(readonly agentId: string, readonly running: number, readonly limit: number) { + super("The agent already runs as many runs at once as it allows."); + this.name = "AgentRunSlotsFull"; + } +} + +export async function assertAgentRunSlotFree(tx: Pick, companyId: string, agentId: string) { + const [slots] = await tx.select({ + limit: maxConcurrentRunsSql(), + running: sql`(select count(*)::int from ${heartbeatRuns} where ${heartbeatRuns.companyId} = ${companyId}::uuid and ${heartbeatRuns.agentId} = ${agentId}::uuid and ${heartbeatRuns.status} = 'running')`, + }).from(agents).where(and(eq(agents.id, agentId), eq(agents.companyId, companyId))); + if (!slots) return; + const running = Number(slots.running); + const limit = Number(slots.limit); + if (running >= limit) throw new AgentRunSlotsFull(agentId, running, limit); +} diff --git a/server/src/services/native-runtime/native-review-dispatch.ts b/server/src/services/native-runtime/native-review-dispatch.ts index 76799a7ebbd..e38367c1d86 100644 --- a/server/src/services/native-runtime/native-review-dispatch.ts +++ b/server/src/services/native-runtime/native-review-dispatch.ts @@ -3,6 +3,7 @@ import { withExternalAdmissionGuard, assertNoExternalOverlap } from "../../modul import { agentWakeupRequests, heartbeatRuns, issues, type Db } from "@paperclipai/db"; import { and, eq } from "drizzle-orm"; import type { PgUpdateSetSource } from "drizzle-orm/pg-core"; +import { assertAgentRunSlotFree } from "../heartbeat/run-slot-admission.js"; import { claimNativeReviewExecutionLock, readNativeReviewAssignmentContext } from "./native-review-participant.js"; /** The run, wake, and issue lock form one reviewer admission claim. */ @@ -35,7 +36,10 @@ export async function claimQueuedNativeReviewRun(db: Db, input: { )).for("update"); if (!run || run.status !== "queued" || run.wakeupRequestId !== input.run.wakeupRequestId) return null; if ((run.contextSnapshot as Record | null)?.issueId !== issueId) return null; - await withExternalAdmissionGuard(tx, run.companyId, run.agentId, () => assertNoExternalOverlap(tx, run.companyId, run.agentId)); + await withExternalAdmissionGuard(tx, run.companyId, run.agentId, async () => { + await assertNoExternalOverlap(tx, run.companyId, run.agentId); + await assertAgentRunSlotFree(tx, run.companyId, run.agentId); + }); const locked = await claimNativeReviewExecutionLock(tx as unknown as Db, { companyId: run.companyId, issueId, agentId: run.agentId, runId: run.id, contextSnapshot: context, agentNameKey: input.agentNameKey, claimedAt: input.claimedAt, diff --git a/server/src/services/workspace-recovery-preparation-dispatch.test.ts b/server/src/services/workspace-recovery-preparation-dispatch.test.ts index e29b2485a84..dd7121d8929 100644 --- a/server/src/services/workspace-recovery-preparation-dispatch.test.ts +++ b/server/src/services/workspace-recovery-preparation-dispatch.test.ts @@ -48,7 +48,7 @@ describe("automatic retained workspace recovery", () => { const experimental = await instanceSettingsService(db).getExperimental(); let stoppedWorker: Awaited> | undefined; let storage: { mockRestore(): void } | undefined; - const starts = vi.spyOn(agentStarts, "withAgentStartLock"); + const starts = vi.spyOn(agentStarts, "dispatchForAgent"); const retryResumes: Promise[] = []; let resumeSpy: { mockRestore(): void } | undefined; let ownedHeartbeat: ReturnType | undefined; @@ -178,7 +178,8 @@ describe("automatic retained workspace recovery", () => { }); const app = boardRecoveryApp(undefined, heartbeat); const gate = new Promise(resolve => { releaseStart = resolve; }); - heldStart = agentStarts.withAgentStartLock(source.agentId, () => gate); + heldStart = agentStarts.dispatchForAgent(source.agentId, { request: null, idle: undefined, merge: () => null, + run: async () => ({ value: await gate }) }); const receipt = mode === "automatic_lost" ? null : (await request(app).post(`/api/issues/${f.issueId}/retry`) .send({ observedRunId: f.runId, idempotencyKey: randomUUID() }).expect(202)).body; if (mode === "automatic_lost") {