diff --git a/server/src/__tests__/recovery-wake-herd.test.ts b/server/src/__tests__/recovery-wake-herd.test.ts index 5025b543e7f..8b6eeee1231 100644 --- a/server/src/__tests__/recovery-wake-herd.test.ts +++ b/server/src/__tests__/recovery-wake-herd.test.ts @@ -309,12 +309,12 @@ suite("recovery wake herd", () => { expect(await executionControlSchedule(db).continuation()).toBeNull(); }, 60_000); - it("leaves a continuation pending while the agent's queue is at its concurrency limit", async () => { + it("leaves a backlog continuation pending while the agent's queue is at its concurrency limit", async () => { const agentId = await seedCompany(1); const busy = await task(agentId); await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: busy.id, invocationSource: "assignment", status: "queued", contextSnapshot: { issueId: busy.id, wakeReason: "issue_assigned" } }); - const record = await pendingContinuation(agentId, 5); + const record = await pendingContinuation(agentId, 120); const wake = fakeWake(); await deliverReconciledExecutions(db, wake as never); expect(wake).not.toHaveBeenCalled(); @@ -355,9 +355,9 @@ suite("recovery wake herd", () => { const fresh = await pendingContinuation(agentId, 1); const wake = fakeWake(); await deliverReconciledExecutions(db, wake as never); - expect(wake).toHaveBeenCalledTimes(1); + expect(wake).toHaveBeenCalledTimes(2); expect(await deliveryOf(held.action.id)).toBe("delivered"); - expect(await deliveryOf(fresh.action.id)).toBe("pending"); + expect(await deliveryOf(fresh.action.id)).toBe("delivered"); }, 60_000); it("admits an older continuation within a bounded number of passes while fresh interruptions keep arriving", async () => { @@ -381,7 +381,7 @@ suite("recovery wake herd", () => { } expect(admittedAt).toBeGreaterThan(0); expect(admittedAt).toBeLessThanOrEqual(bound); - expect(wake).toHaveBeenCalledTimes(1); + expect(wake.mock.calls.filter(call => call[1].contextSnapshot.recoveryActionId === old.action.id)).toHaveLength(1); } finally { vi.useRealTimers(); } @@ -394,7 +394,7 @@ suite("recovery wake herd", () => { const busy = await task(agentId); await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: busy.id, invocationSource: "assignment", status: "queued", contextSnapshot: { issueId: busy.id, wakeReason: "issue_assigned" } }); - for (let index = 0; index < 8; index += 1) await pendingContinuation(agentId, 2 + index); + for (let index = 0; index < 8; index += 1) await pendingContinuation(agentId, 60 + index); } const idleAgent = await addAgent(1); const idle = await pendingContinuation(idleAgent, 600); @@ -474,7 +474,7 @@ suite("recovery wake herd", () => { } const sameAgentStuck = await pendingContinuation(agentId, 300); await holdPastBound(sameAgentStuck, 60 * 60_000); - const waiting = await pendingContinuation(agentId, 10); + const waiting = await pendingContinuation(agentId, 60); await holdPastBound(waiting); const stuckIds = new Set([...stuck, sameAgentStuck].map(record => record.action.id)); const created = fakeWake(); @@ -507,7 +507,7 @@ suite("recovery wake herd", () => { await deliverReconciledExecutions(db, wake as never); } expect(await deliveryOf(overdue.action.id)).toBe("delivered"); - expect(created.mock.calls[0]![1].contextSnapshot.recoveryActionId).toBe(overdue.action.id); + expect(created.mock.calls.map(call => call[1].contextSnapshot.recoveryActionId)).toContain(overdue.action.id); }, 60_000); it("delivers another agent's continuation in one pass while one agent refuses a backlog of more than 200", async () => { @@ -592,4 +592,114 @@ suite("recovery wake herd", () => { expect(wake).toHaveBeenCalledTimes(1); expect(await deliveryOf(held.action.id)).toBe("delivered"); }, 60_000); + async function queueOrdinaryWork(agentId: string, count: number) { + for (let index = 0; index < count; index += 1) { + const busy = await task(agentId); + await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: busy.id, invocationSource: "assignment", status: "queued", + contextSnapshot: { issueId: busy.id, wakeReason: "issue_assigned" } }); + } + } + const deliveredIds = (wake: ReturnType) => + wake.mock.calls.map(call => String(call[1].contextSnapshot.recoveryActionId)); + + it("enqueues a fresh interruption's continuation at once while the agent's queue is deeper than its concurrency limit", async () => { + const agentId = await seedCompany(1); + await queueOrdinaryWork(agentId, 3); + const record = await pendingContinuation(agentId, 3); + const wake = fakeWake(); + await deliverReconciledExecutions(db, wake as never); + expect(deliveredIds(wake)).toEqual([record.action.id]); + expect(await deliveryOf(record.action.id)).toBe("delivered"); + }, 60_000); + + it("delivers a fresh interruption ahead of an hours-old backlog for the same agent and keeps draining the backlog", async () => { + const agentId = await seedCompany(2); + await queueOrdinaryWork(agentId, 4); + const backlog: Array>> = []; + for (let index = 0; index < 4; index += 1) { + const record = await pendingContinuation(agentId, 300 + index); + await holdPastBound(record, (4 * 60 - index) * 60_000); + backlog.push(record); + } + const fresh = await pendingContinuation(agentId, 3); + const wake = fakeWake(); + await deliverReconciledExecutions(db, wake as never); + expect(deliveredIds(wake)).toEqual([fresh.action.id, backlog[0]!.action.id]); + for (let pass = 0; pass < backlog.length; pass += 1) await deliverReconciledExecutions(db, wake as never); + expect(await Promise.all(backlog.map(record => deliveryOf(record.action.id)))).toEqual(backlog.map(() => "delivered")); + expect(wake).toHaveBeenCalledTimes(backlog.length + 1); + }, 60_000); + + it("enqueues many fresh continuations for one agent one per pass, newest first, and leaves none behind", async () => { + const agentId = await seedCompany(1); + await queueOrdinaryWork(agentId, 2); + const records: Array>> = []; + for (let index = 0; index < 5; index += 1) records.push(await pendingContinuation(agentId, 1 + index)); + const wake = fakeWake(); + for (let pass = 0; pass < records.length; pass += 1) { + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(pass + 1); + } + expect(deliveredIds(wake)).toEqual(records.map(record => record.action.id)); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(records.length); + }, 60_000); + + it("keeps a share of each pass for backlog while fresh interruptions fill the company cap", async () => { + const freshAgents = [await seedCompany(1)]; + for (let index = 1; index < RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS + 2; index += 1) freshAgents.push(await addAgent(1)); + for (const agentId of freshAgents) await pendingContinuation(agentId, 2); + const backlogAgent = await addAgent(1); + const backlog = await pendingContinuation(backlogAgent, 240); + const wake = fakeWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS); + expect(await deliveryOf(backlog.action.id)).toBe("delivered"); + expect(wake.mock.calls.filter(call => call[0] !== backlogAgent)).toHaveLength(RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS - 1); + }, 60_000); + + it("delivers a fresh continuation for every agent up to the company cap in one pass when no backlog waits", async () => { + const agentIds = [await seedCompany(1)]; + for (let index = 1; index < RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS; index += 1) agentIds.push(await addAgent(1)); + const records = []; + for (const agentId of agentIds) records.push(await pendingContinuation(agentId, 2)); + const wake = fakeWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS); + expect(await Promise.all(records.map(record => deliveryOf(record.action.id)))).toEqual(records.map(() => "delivered")); + }, 60_000); + + it("keeps a fresh continuation whose wake was refused pending and enqueues it on the next pass", async () => { + const agentId = await seedCompany(1); + await queueOrdinaryWork(agentId, 3); + const record = await pendingContinuation(agentId, 2); + const created = fakeWake(); + let refusing = true; + const wake = vi.fn(async (wakeAgentId: string, opts: { contextSnapshot: Record }) => + refusing ? null : created(wakeAgentId, opts)); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + expect(await deliveryOf(record.action.id)).toBe("pending"); + refusing = false; + await deliverReconciledExecutions(db, wake as never); + expect(created).toHaveBeenCalledTimes(1); + expect(await deliveryOf(record.action.id)).toBe("delivered"); + }, 60_000); + it("records why a system wake was folded into the task's pending continuation instead of leaving no trace", async () => { + const agentId = await seedCompany(1); + const record = await pendingContinuation(agentId, 30); + const heartbeat = heartbeatService(db); + try { + const run = await heartbeat.wakeup(agentId, { source: "automation", triggerDetail: "system", reason: "issue_continuation_needed", + payload: { issueId: record.issue.id, mutation: "assigned_work_invariant" }, requestedByActorType: "system", + contextSnapshot: { issueId: record.issue.id, taskId: record.issue.id, wakeReason: "issue_continuation_needed" } }); + expect(run).toBeNull(); + const rows = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.agentId, agentId), + sql`${agentWakeupRequests.payload}->>'issueId' = ${record.issue.id}`)); + expect(rows).toMatchObject([{ status: "skipped", reason: "issue_continuation_pending" }]); + expect(await deliveryOf(record.action.id)).toBe("pending"); + } finally { + await heartbeat.drainActiveRunExecutions(); + } + }, 60_000); }); diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index caa6016fa73..b543a13108c 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -40,10 +40,10 @@ import { provenGoneLegacyControllerBootId } from "./legacy-controller-lease.js"; import { continueFreshInTransaction, continueHeldExecutions, freshContinuationCheckDue, recordFreshContinuationCheck, settleCoalescedFreshContinuationHolds } from "./fresh-continuation-fallback.js"; import { alias } from "drizzle-orm/pg-core"; -import { RECOVERY_WAKE_REASONS } from "./heartbeat/queued-run-dispatch-order.js"; +import { RECOVERY_BACKLOG_AGE_MS, RECOVERY_WAKE_REASONS } from "./heartbeat/queued-run-dispatch-order.js"; import { parseHeartbeatPolicy } from "./heartbeat/timer-policy.js"; -import { RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS, RECOVERY_CONTINUATION_MAX_HOLD_MS, +import { RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS, RECOVERY_CONTINUATION_BACKLOG_SHARE_PER_PASS, RECOVERY_CONTINUATION_MAX_HOLD_MS, recoveryContinuationHeldSince } from "./recovery-continuation-pacing.js"; export { RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS, RECOVERY_CONTINUATION_DELIVERY_BOUND_MS, @@ -370,18 +370,25 @@ export async function deliverReconciledExecutions( const recoveryReasons = [...RECOVERY_WAKE_REASONS]; const sourceRun = alias(heartbeatRuns, "continuation_source_run"); const seenActionIds = new Set(); - const seenAgentIds = new Set(); + const seenAgentIds: Record> = { fresh: new Set(), backlog: new Set() }; const admittedByCompany = new Map(); + let companyLimit = RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS; const fullCompanies = () => [...admittedByCompany].flatMap(([companyId, count]) => - count >= RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS ? [companyId] : []); + count >= companyLimit ? [companyId] : []); const heldSince = sql`coalesce(${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}', ${issueRecoveryActions.resolvedAt}::text, ${issueRecoveryActions.updatedAt}::text)::timestamptz`; - const readPendingPage = async () => { + const attemptedAt = sql`(${issueRecoveryActions.evidence}->>'continuationDeliveryAttemptedAt')::timestamptz`; + const freshSource = () => sql`coalesce(${sourceRun.finishedAt} > ${new Date(Date.now() - RECOVERY_BACKLOG_AGE_MS).toISOString()}::timestamptz, false)`; + const readPendingPage = async (kind: ContinuationClass) => { const companies = fullCompanies(); + const seenAgents = seenAgentIds[kind]; const heldBound = new Date(Date.now() - RECOVERY_CONTINUATION_MAX_HOLD_MS).toISOString(); const pastBound = sql`${heldSince} <= ${heldBound}::timestamptz`; - const overdueTurn = sql`case when ${pastBound} then greatest(${heldSince}, - (${issueRecoveryActions.evidence}->>'continuationDeliveryAttemptedAt')::timestamptz) end`; - const priority = [ + const overdueTurn = sql`case when ${pastBound} then greatest(${heldSince}, ${attemptedAt}) end`; + const priority = kind === "fresh" ? [ + sql`${attemptedAt} asc nulls first`, + sql`${sourceRun.finishedAt} desc`, + sql`${issueRecoveryActions.updatedAt} asc`, + ] : [ sql`${pastBound} desc`, sql`${overdueTurn} asc nulls last`, sql`${sourceRun.finishedAt} desc nulls last`, @@ -393,6 +400,7 @@ export async function deliverReconciledExecutions( pastBound: sql`${pastBound}`.as("past_bound"), heldAt: sql`${overdueTurn}`.as("held_at"), sourceFinishedAt: sql`${sourceRun.finishedAt}`.as("source_finished_at"), + attemptedAt: sql`${attemptedAt}`.as("attempted_at"), actionUpdatedAt: sql`${issueRecoveryActions.updatedAt}`.as("action_updated_at"), agentRank: sql`row_number() over ( partition by coalesce(${issueRecoveryActions.returnOwnerAgentId}::text, ${issueRecoveryActions.id}::text) @@ -409,9 +417,10 @@ export async function deliverReconciledExecutions( eq(issueRecoveryActions.status, "resolved"), sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, seenActionIds.size ? not(inArray(issueRecoveryActions.id, [...seenActionIds])) : undefined, - seenAgentIds.size ? sql`coalesce(${issueRecoveryActions.returnOwnerAgentId}::text, '') <> all(${sql.param([...seenAgentIds])}::text[])` : undefined, + seenAgents.size ? sql`coalesce(${issueRecoveryActions.returnOwnerAgentId}::text, '') <> all(${sql.param([...seenAgents])}::text[])` : undefined, companies.length ? not(inArray(issueRecoveryActions.companyId, companies)) : undefined, - sql`(${pastBound} or not exists (select 1 from ${heartbeatRuns} queued_recovery + kind === "fresh" ? freshSource() : sql`not ${freshSource()}`, + kind === "fresh" ? undefined : sql`(${pastBound} or not exists (select 1 from ${heartbeatRuns} queued_recovery where queued_recovery.company_id = ${issueRecoveryActions.companyId} and queued_recovery.agent_id = ${issueRecoveryActions.returnOwnerAgentId} and queued_recovery.status = 'queued' @@ -424,12 +433,16 @@ export async function deliverReconciledExecutions( .from(candidates) .innerJoin(issueRecoveryActions, eq(issueRecoveryActions.id, candidates.id)) .where(eq(candidates.agentRank, 1)) - .orderBy( + .orderBy(...(kind === "fresh" ? [ + sql`${candidates.attemptedAt} asc nulls first`, + sql`${candidates.sourceFinishedAt} desc`, + sql`${candidates.actionUpdatedAt} asc`, + ] : [ sql`${candidates.pastBound} desc`, sql`${candidates.heldAt} asc nulls last`, sql`${candidates.sourceFinishedAt} desc nulls last`, sql`${candidates.actionUpdatedAt} asc`, - ) + ])) .limit(25); return rows.map(row => row.action); }; @@ -442,8 +455,9 @@ export async function deliverReconciledExecutions( eq(issueRecoveryActions.status, "resolved"), sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, )); - const agentHasRoom = async (action: typeof issueRecoveryActions.$inferSelect, agentId: string) => { - if ((admittedByCompany.get(action.companyId) ?? 0) >= RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS) return false; + const agentHasRoom = async (action: typeof issueRecoveryActions.$inferSelect, agentId: string, kind: ContinuationClass) => { + if ((admittedByCompany.get(action.companyId) ?? 0) >= companyLimit) return false; + if (kind === "fresh") return true; const [agent] = await db.select().from(agents).where(and(eq(agents.companyId, action.companyId), eq(agents.id, agentId))); if (!agent) return true; const [queued] = await db.select({ count: sql`count(*)::int` }).from(heartbeatRuns).where(and( @@ -451,12 +465,23 @@ export async function deliverReconciledExecutions( if (Number(queued?.count ?? 0) < parseHeartbeatPolicy(agent).maxConcurrentRuns) return true; return Date.now() - recoveryContinuationHeldSince(action) >= RECOVERY_CONTINUATION_MAX_HOLD_MS; }; - for (let page = 0; page < RECOVERY_CONTINUATION_PAGES_PER_PASS; page += 1) { - const pending = await readPendingPage(); - if (!pending.length) break; + const cap = RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS; + const phases: Array<[ContinuationClass, number]> = [ + ["fresh", cap - RECOVERY_CONTINUATION_BACKLOG_SHARE_PER_PASS], ["backlog", cap], ["fresh", cap]]; + for (let page = 0; page < RECOVERY_CONTINUATION_PAGES_PER_PASS * phases.length; page += 1) { + const phase = Math.floor(page / RECOVERY_CONTINUATION_PAGES_PER_PASS); + const [kind, limit] = phases[phase]!; + companyLimit = limit; + const seenAgents = seenAgentIds[kind]; + const pending = await readPendingPage(kind); + if (!pending.length) { + page = (phase + 1) * RECOVERY_CONTINUATION_PAGES_PER_PASS - 1; + continue; + } for (const action of pending) { + if ((admittedByCompany.get(action.companyId) ?? 0) >= companyLimit) continue; seenActionIds.add(action.id); - if (action.returnOwnerAgentId && seenAgentIds.has(action.returnOwnerAgentId)) continue; + if (action.returnOwnerAgentId && seenAgents.has(action.returnOwnerAgentId)) continue; try { const decision = action.evidence.executionReconciliation as ExecutionReconciliation | undefined; @@ -510,8 +535,8 @@ export async function deliverReconciledExecutions( continue; } } - if (!await agentHasRoom(action, action.returnOwnerAgentId)) { - seenAgentIds.add(action.returnOwnerAgentId); + if (!await agentHasRoom(action, action.returnOwnerAgentId, kind)) { + seenAgents.add(action.returnOwnerAgentId); continue; } const run = await wake(action.returnOwnerAgentId, { @@ -535,7 +560,7 @@ export async function deliverReconciledExecutions( ? { freshContinuationIsolation: action.id } : {}), }, }); - seenAgentIds.add(action.returnOwnerAgentId); + seenAgents.add(action.returnOwnerAgentId); if (!run) { await deferRefusedContinuation(action); continue; @@ -569,7 +594,7 @@ export async function deliverReconciledExecutions( delivery: "delivered", continuationRunId: run.id }); }); } catch { - if (action.returnOwnerAgentId) seenAgentIds.add(action.returnOwnerAgentId); + if (action.returnOwnerAgentId) seenAgents.add(action.returnOwnerAgentId); await deferRefusedContinuation(action).catch(() => undefined); logger.warn( { recoveryActionId: action.id }, @@ -580,6 +605,8 @@ export async function deliverReconciledExecutions( } } +type ContinuationClass = "fresh" | "backlog"; + const supersedingWorkspaceHold = alias(issueRecoveryActions, "superseding_workspace_hold"); function supersedingNativeWorkspaceHold(db: Db, input: { diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 63fc3898c87..a0a15712898 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -4018,8 +4018,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende && !executionReconciliationWake; // A recorded fresh continuation that has no run yet is this task's // next run: another system wake is coalesced into it, never a second run. - if (systemWake && !executionBlocker && await hasPendingFreshContinuation(tx as unknown as Db, issue.companyId, issue.id)) + if (systemWake && !executionBlocker && await hasPendingFreshContinuation(tx as unknown as Db, issue.companyId, issue.id)) { + await tx.insert(agentWakeupRequests).values({ + ...durableReceiptFields, companyId: agent.companyId, agentId, source, triggerDetail, + reason: "issue_continuation_pending", payload, status: "skipped", + requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, finishedAt: new Date(), + }); return { kind: "skipped" as const }; + } if ((executionBlocker || opts.queuedCommentInterruptId) && !ownsReconstructionContinuation && !(await admitExplicitNativeContinuation({ db: tx as unknown as Db, companyId: issue.companyId, issueId: issue.id, agentId, actorType: taskRetryActorId ? "user" : opts.requestedByActorType, actorId: taskRetryActorId ?? opts.requestedByActorId, diff --git a/server/src/services/recovery-continuation-pacing.ts b/server/src/services/recovery-continuation-pacing.ts index d0d9b9e18c2..33e1595f88e 100644 --- a/server/src/services/recovery-continuation-pacing.ts +++ b/server/src/services/recovery-continuation-pacing.ts @@ -1,4 +1,5 @@ export const RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS = 5; +export const RECOVERY_CONTINUATION_BACKLOG_SHARE_PER_PASS = 1; export const RECOVERY_CONTINUATION_MAX_HOLD_MS = 5 * 60_000; export const RECOVERY_CONTINUATION_DELIVERY_BOUND_MS = 15 * 60_000; export const PENDING_CONTINUATION_BOUND_MS = RECOVERY_CONTINUATION_MAX_HOLD_MS + 2 * 60_000;