diff --git a/server/src/__tests__/recovery-wake-herd.test.ts b/server/src/__tests__/recovery-wake-herd.test.ts index ff58c96927b..5edf8550679 100644 --- a/server/src/__tests__/recovery-wake-herd.test.ts +++ b/server/src/__tests__/recovery-wake-herd.test.ts @@ -158,6 +158,106 @@ suite("recovery wake herd", () => { } }, 120_000); + async function continuationOfInterruptedPersonRetry(agentId: string, createdMinutesAgo: number) { + const issue = await task(agentId); + const finishedAt = new Date(Date.now() - (createdMinutesAgo + 2) * 60_000); + const [retry] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "on_demand", triggerDetail: "manual", + reason: "task_retry", status: "queued", requestedByActorType: "user", requestedByActorId: userId, + payload: { issueId: issue.id } }).returning(); + const [source] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: issue.id, invocationSource: "on_demand", + triggerDetail: "manual", status: "interrupted", errorCode: "server_shutdown_interrupted", wakeupRequestId: retry!.id, + responsibleUserId: userId, startedAt: new Date(finishedAt.getTime() - 60_000), finishedAt, + createdAt: new Date(finishedAt.getTime() - 120_000), + contextSnapshot: { issueId: issue.id, wakeReason: "task_retry", taskRetryRequestId: retry!.id } }).returning(); + await db.update(agentWakeupRequests).set({ runId: source!.id, status: "completed", finishedAt }) + .where(eq(agentWakeupRequests.id, retry!.id)); + const [continuation] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: issue.id, + invocationSource: "automation", triggerDetail: "system", status: "queued", responsibleUserId: userId, + createdAt: new Date(Date.now() - createdMinutesAgo * 60_000), + contextSnapshot: { issueId: issue.id, taskId: issue.id, wakeReason: "issue_continuation_needed", previousRunId: source!.id, + source: "issue.assigned_work_invariant", assignedWorkViolation: "retry_offered" } }).returning(); + return continuation!; + } + + it("starts the system continuation of a person's interrupted Retry ahead of fresher system work", async () => { + const agentId = await seedCompany(1); + const continuation = await continuationOfInterruptedPersonRetry(agentId, 40); + const fresher: Array = []; + for (let index = 0; index < 3; index += 1) fresher.push(await queuedSystemRun(agentId, 30 - index, "issue_assigned")); + fresher.push(await queuedRecoveryRun(agentId, 2, 3)); + const heartbeat = heartbeatService(db); + try { + await heartbeat.resumeQueuedRuns(); + expect((await readRun(continuation.id)).startedAt).not.toBeNull(); + for (const run of fresher) expect(await readRun(run.id)).toMatchObject({ status: "queued", startedAt: null }); + } finally { + await heartbeat.drainActiveRunExecutions(); + } + }, 120_000); + + it("keeps a person's Retry first after its system continuation is interrupted again", async () => { + const agentId = await seedCompany(1); + const first = await continuationOfInterruptedPersonRetry(agentId, 90); + const firstEndedAt = new Date(Date.now() - 45 * 60_000); + await db.update(heartbeatRuns).set({ status: "interrupted", errorCode: "server_shutdown_interrupted", + startedAt: new Date(firstEndedAt.getTime() - 60_000), finishedAt: firstEndedAt }).where(eq(heartbeatRuns.id, first.id)); + const [second] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: first.issueId, + invocationSource: "automation", triggerDetail: "system", status: "queued", responsibleUserId: userId, + createdAt: new Date(Date.now() - 40 * 60_000), + contextSnapshot: { issueId: first.issueId, taskId: first.issueId, wakeReason: "issue_continuation_needed", + previousRunId: first.id, source: "issue.assigned_work_invariant" } }).returning(); + const fresher: Array = []; + for (let index = 0; index < 3; index += 1) fresher.push(await queuedSystemRun(agentId, 30 - index, "issue_assigned")); + const heartbeat = heartbeatService(db); + try { + await heartbeat.resumeQueuedRuns(); + expect((await readRun(second!.id)).startedAt).not.toBeNull(); + for (const run of fresher) expect(await readRun(run.id)).toMatchObject({ status: "queued", startedAt: null }); + } finally { + await heartbeat.drainActiveRunExecutions(); + } + }, 120_000); + + it("starts a system continuation that a person's Retry coalesced into ahead of fresher system work", async () => { + const agentId = await seedCompany(1); + const continuation = await queuedSystemRun(agentId, 40, "issue_continuation_needed"); + await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "on_demand", triggerDetail: "manual", + reason: "task_retry", status: "coalesced", coalescedCount: 1, requestedByActorType: "user", requestedByActorId: userId, + runId: continuation.id, finishedAt: new Date(), payload: { issueId: continuation.issueId } }); + const fresher: Array = []; + for (let index = 0; index < 3; index += 1) fresher.push(await queuedSystemRun(agentId, 30 - index, "issue_assigned")); + const heartbeat = heartbeatService(db); + try { + await heartbeat.resumeQueuedRuns(); + expect((await readRun(continuation.id)).startedAt).not.toBeNull(); + for (const run of fresher) expect(await readRun(run.id)).toMatchObject({ status: "queued", startedAt: null }); + } finally { + await heartbeat.drainActiveRunExecutions(); + } + }, 120_000); + + it("gives the longest-waiting queued run a free slot after three fresher starts passed it over", async () => { + const agentId = await seedCompany(1); + const old = await queuedRecoveryRun(agentId, 150, 152); + const people: Array = []; + for (let index = 0; index < 3; index += 1) people.push(await queuedPersonRetry(agentId)); + const issue = await task(agentId); + for (let index = 0; index < 3; index += 1) { + const startedAt = new Date(Date.now() - (3 - index) * 60_000); + await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: issue.id, invocationSource: "on_demand", + triggerDetail: "manual", status: "succeeded", createdAt: new Date(startedAt.getTime() - 1_000), startedAt, + finishedAt: new Date(startedAt.getTime() + 1_000), contextSnapshot: { issueId: issue.id, wakeReason: "issue_recovery_action_restored" } }); + } + const heartbeat = heartbeatService(db); + try { + await heartbeat.resumeQueuedRuns(); + expect((await readRun(old.id)).startedAt).not.toBeNull(); + for (const run of people) expect(await readRun(run.id)).toMatchObject({ status: "queued", startedAt: null }); + } finally { + await heartbeat.drainActiveRunExecutions(); + } + }, 120_000); + async function pendingContinuation(agentId: string, endedMinutesAgo: number) { const issue = await task(agentId, "todo"); const source = await sourceRun(agentId, issue.id, endedMinutesAgo); diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index ec362ee5f17..63fc3898c87 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -10,7 +10,8 @@ import { hasAutomaticInteractionDeliveryHold, INTERACTION_DELIVERY_DENIED, reche import { deferredWorkIntentWait } from "../work-intent-outbox.js"; import { notifyIssueRecovery } from "../issue-recovery-notifications.js"; import { createHeartbeatTimerPolicy } from "./timer-policy.js"; -import { RECOVERY_DISPATCH_SHARE, isRecoveryWakeReason, isSystemRecoveryRun, createQueuedRunDispatcher } from "./queued-run-dispatch-order.js"; +import { OLDEST_DISPATCH_SHARE, PERSON_WAKE_REASONS, QUEUED_RUN_AGING_MS, RECOVERY_DISPATCH_SHARE, createQueuedRunDispatcher, + hasQueuedRunAged, isRecoveryWakeReason, isSystemRecoveryRun, oldestReadyQueuedRun } from "./queued-run-dispatch-order.js"; import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { PersistedExecutionWaitRefusal, writeDeferredExecutionWait } from "../deferred-execution-wait.js"; import { assignmentWakePayloadSql, isAssignmentWakePayload, notifyDeferredExecutionWork } from "../deferred-execution-notifications.js"; @@ -372,6 +373,8 @@ export function shouldQueueFollowupForRunningIssueWake(input: { /** Agent Chat outbox receipts. The outbox, not a skipped receipt, owns a capped message. */ const CONVERSATION_COMMENT_KEY_PREFIX = "conversation-comment:"; +const RUN_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const PERSON_ORIGIN_CHAIN_DEPTH = 8; function dailyLimitConversationRefusal(cap: { reason: string; observed: number; limit: number }) { // No receipt is written, so the person's message stays owed in the outbox // until the limit resets (agent-conversations.ts conversationDeliveryHold). @@ -2346,6 +2349,48 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende .catch(() => logger.warn({ agentId: agent.id }, "Could not attach queued agent setup connection request")); } + const declinedAgedClaimAt = new Map(); + + function noteDeclinedAgedClaim(run: typeof heartbeatRuns.$inferSelect, reason: string) { + const now = Date.now(); + if (!hasQueuedRunAged(run, new Date(now))) return; + declinedAgedClaimAt.set(run.id, now); + logger.warn({ runId: run.id, agentId: run.agentId, companyId: run.companyId, reason, + queuedForMs: now - run.createdAt.getTime() }, + "long-queued heartbeat run was offered a free slot but could not be claimed; it stays queued and the next oldest run takes its turn"); + } + + function pruneDeclinedAgedClaims(now: Date) { + for (const [id, at] of declinedAgedClaimAt) { + if (now.getTime() - at >= QUEUED_RUN_AGING_MS) declinedAgedClaimAt.delete(id); + } + } + + async function listPersonOriginSourceRunIds(companyId: string, sourceRunIds: string[]) { + const uuidPattern = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; + const result = await db.execute<{ root_id: string }>(sql` + with recursive chain(root_id, run_id, depth) as ( + select source.id, source.id, 0 from ${heartbeatRuns} source + where source.company_id = ${companyId} and source.id in (${sql.join(sourceRunIds.map(id => sql`${id}::uuid`), sql`, `)}) + union all + select chain.root_id, parent.id, chain.depth + 1 + from chain + join ${heartbeatRuns} child on child.id = chain.run_id + join ${heartbeatRuns} parent on parent.company_id = ${companyId} and parent.id = coalesce( + case when child.context_snapshot->>'previousRunId' ~* ${uuidPattern} + then (child.context_snapshot->>'previousRunId')::uuid end, + child.retry_of_run_id) + where chain.depth < ${PERSON_ORIGIN_CHAIN_DEPTH} + ) + select distinct chain.root_id from chain + join ${heartbeatRuns} run on run.id = chain.run_id + where run.context_snapshot->>'wakeReason' in (${sql.join([...PERSON_WAKE_REASONS].map(reason => sql`${reason}`), sql`, `)}) + or run.context_snapshot->>'taskRetryRequestId' is not null + or exists (select 1 from ${agentWakeupRequests} wake where wake.company_id = ${companyId} + and wake.requested_by_actor_type = 'user' and (wake.run_id = run.id or wake.id = run.wakeup_request_id))`); + return [...result].map(row => String(row.root_id)); + } + async function queuedRunDispatcherForAgent( agent: typeof agents.$inferSelect, queuedRuns: Array, @@ -2353,6 +2398,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende issueById: Map, ) { const now = new Date(); + pruneDeclinedAgedClaims(now); const wakeReasonOf = (run: typeof heartbeatRuns.$inferSelect) => readNonEmptyString(parseObject(run.contextSnapshot).wakeReason); const requestIds = [...new Set(queuedRuns.flatMap(run => run.wakeupRequestId ? [run.wakeupRequestId] : []))]; @@ -2368,30 +2414,29 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende eq(agentWakeupRequests.agentId, agent.id), eq(agentWakeupRequests.requestedByActorType, "user"))) : []; const userRunIds = new Set(joinedUserRequests.flatMap(row => row.runId ? [row.runId] : [])); - const requestedByUser = (run: typeof heartbeatRuns.$inferSelect) => + const joinedByUser = (run: typeof heartbeatRuns.$inferSelect) => userRunIds.has(run.id) || Boolean(run.wakeupRequestId && userRequestIds.has(run.wakeupRequestId)); const sourceRunIdOf = (run: typeof heartbeatRuns.$inferSelect) => readNonEmptyString(parseObject(run.contextSnapshot).previousRunId) ?? run.retryOfRunId ?? null; - const recoveryRuns = queuedRuns.filter(run => isSystemRecoveryRun({ wakeReason: wakeReasonOf(run), - requestedByUser: requestedByUser(run) })); - const sourceRunIds = [...new Set(recoveryRuns.flatMap(run => sourceRunIdOf(run) ?? []))]; - const sourceRows = queuedRuns.length > 1 && sourceRunIds.length + const sourceRunIds = queuedRuns.length > 1 + ? [...new Set(queuedRuns.flatMap(run => { + const sourceRunId = joinedByUser(run) ? null : sourceRunIdOf(run); + return sourceRunId && RUN_ID_PATTERN.test(sourceRunId) ? [sourceRunId] : []; + }))] + : []; + const sourceRows = sourceRunIds.length ? await db.select({ id: heartbeatRuns.id, finishedAt: heartbeatRuns.finishedAt }).from(heartbeatRuns) .where(and(eq(heartbeatRuns.companyId, agent.companyId), inArray(heartbeatRuns.id, sourceRunIds))) : []; const sourceEndedAt = new Map(sourceRows.map(row => [row.id, row.finishedAt])); - let startsSinceRecovery = 0; - if (recoveryRuns.length && recoveryRuns.length < queuedRuns.length) { - const oldestRecoveryAt = new Date(Math.min(...recoveryRuns.map(run => run.createdAt.getTime()))); - const recentStarts = await db.select({ wakeReason: sql`${heartbeatRuns.contextSnapshot}->>'wakeReason'` }) - .from(heartbeatRuns) - .where(and(eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id), - isNotNull(heartbeatRuns.startedAt), gte(heartbeatRuns.startedAt, oldestRecoveryAt))) - .orderBy(desc(heartbeatRuns.startedAt)) - .limit(RECOVERY_DISPATCH_SHARE); - const lastRecovery = recentStarts.findIndex(row => isRecoveryWakeReason(row.wakeReason)); - startsSinceRecovery = lastRecovery < 0 ? recentStarts.length : lastRecovery; - } + const personSourceRunIds = new Set(sourceRows.length + ? await listPersonOriginSourceRunIds(agent.companyId, sourceRows.map(row => row.id)) + : []); + const requestedByUser = (run: typeof heartbeatRuns.$inferSelect) => { + if (joinedByUser(run)) return true; + const sourceRunId = sourceRunIdOf(run); + return Boolean(sourceRunId && personSourceRunIds.has(sourceRunId)); + }; const facts = queuedRuns.map(run => { const issueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); const ready = issueId ? (dependencyReadiness.get(issueId)?.isDependencyReady ?? true) : true; @@ -2406,9 +2451,38 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende sourceEndedAt: sourceRunId ? sourceEndedAt.get(sourceRunId) ?? null : null, readinessRank: issueId ? (ready ? (issue?.status === "in_progress" ? 0 : 1) : 3) : 2, priorityRank: issueRunPriorityRank(issue?.priority), + shareDeferred: declinedAgedClaimAt.has(run.id), }; }); - const dispatcher = createQueuedRunDispatcher(facts, { now, startsSinceRecovery }); + const recoveryRuns = facts.filter(entry => isSystemRecoveryRun(entry)); + const oldestReady = oldestReadyQueuedRun(facts); + const agedOldest = oldestReady && hasQueuedRunAged(oldestReady, now) ? oldestReady : null; + const countsRecovery = recoveryRuns.length > 0 && recoveryRuns.length < facts.length; + let startsSinceRecovery = 0; + let startsSinceOldest = 0; + if (countsRecovery || (agedOldest && facts.length > 1)) { + const oldestRecoveryAt = countsRecovery ? Math.min(...recoveryRuns.map(entry => entry.createdAt.getTime())) : Infinity; + const since = new Date(Math.min(oldestRecoveryAt, agedOldest?.createdAt.getTime() ?? Infinity)); + const recentStarts = await db.select({ wakeReason: sql`${heartbeatRuns.contextSnapshot}->>'wakeReason'`, + startedAt: heartbeatRuns.startedAt, createdAt: heartbeatRuns.createdAt }) + .from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id), + isNotNull(heartbeatRuns.startedAt), gte(heartbeatRuns.startedAt, since))) + .orderBy(desc(heartbeatRuns.startedAt)) + .limit(Math.max(RECOVERY_DISPATCH_SHARE, OLDEST_DISPATCH_SHARE)); + const startsSince = (at: number) => recentStarts.filter(row => row.startedAt && row.startedAt.getTime() >= at); + if (countsRecovery) { + const starts = startsSince(oldestRecoveryAt).slice(0, RECOVERY_DISPATCH_SHARE); + const lastRecovery = starts.findIndex(row => isRecoveryWakeReason(row.wakeReason)); + startsSinceRecovery = lastRecovery < 0 ? starts.length : lastRecovery; + } + if (agedOldest && facts.length > 1) { + const starts = startsSince(agedOldest.createdAt.getTime()).slice(0, OLDEST_DISPATCH_SHARE); + const lastOldest = starts.findIndex(row => row.createdAt.getTime() <= agedOldest.createdAt.getTime()); + startsSinceOldest = lastOldest < 0 ? starts.length : lastOldest; + } + } + const dispatcher = createQueuedRunDispatcher(facts, { now, startsSinceRecovery, startsSinceOldest }); const byRun = new Map(facts.map(entry => [entry.run, entry])); return { next: () => dispatcher.next()?.run ?? null, @@ -2510,11 +2584,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende { err, runId: queuedRun.id, agentId: queuedRun.agentId, companyId: queuedRun.companyId }, "queued heartbeat run claim was rejected; leaving it queued for the next recovery pass", ); + noteDeclinedAgedClaim(queuedRun, err.message); continue; } if (claimed) { claimedRuns.push(claimed); dispatcher.claimed(queuedRun); + declinedAgedClaimAt.delete(queuedRun.id); + } else { + noteDeclinedAgedClaim(queuedRun, "claim_declined"); } } if (claimedRuns.length === 0) return []; diff --git a/server/src/services/heartbeat/queued-run-dispatch-order.test.ts b/server/src/services/heartbeat/queued-run-dispatch-order.test.ts index ae74437a1fb..1abcb6b1818 100644 --- a/server/src/services/heartbeat/queued-run-dispatch-order.test.ts +++ b/server/src/services/heartbeat/queued-run-dispatch-order.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it } from "vitest"; import { + OLDEST_DISPATCH_SHARE, + QUEUED_RUN_AGING_MS, RECOVERY_BACKLOG_AGE_MS, createQueuedRunDispatcher, RECOVERY_DISPATCH_SHARE, @@ -122,4 +124,87 @@ describe("queued run dispatch order", () => { const ordered = orderQueuedRunsForDispatch([todo, later, earlier, urgent, inProgress], { now, startsSinceRecovery: 0 }); expect(ordered.map(entry => entry.id)).toEqual([urgent.id, earlier.id, later.id, inProgress.id, todo.id]); }); + + it("starts an old queued continuation within a bounded number of starts while fresh person and system work keeps arriving", () => { + const continuation = run({ wakeReason: "issue_continuation_needed", createdAt: minutesAgo(150), sourceEndedAt: minutesAgo(152) }); + const started: QueuedRunDispatchFacts[] = []; + let pending: QueuedRunDispatchFacts[] = [continuation]; + const steps = 40; + for (let step = 0; step < steps; step += 1) { + pending.push(personRetry(), run({ wakeReason: "issue_assigned", createdAt: minutesAgo(1) }), + run({ wakeReason: "issue_continuation_needed", createdAt: minutesAgo(1), sourceEndedAt: minutesAgo(2) })); + const recent = started.slice(-RECOVERY_DISPATCH_SHARE).reverse(); + const lastRecovery = recent.findIndex(entry => isRecoveryWakeReason(entry.wakeReason)); + const oldest = pending.reduce((left, right) => left.createdAt <= right.createdAt ? left : right); + const recentForOldest = started.slice(-OLDEST_DISPATCH_SHARE).reverse(); + const lastOldest = recentForOldest.findIndex(entry => entry.createdAt <= oldest.createdAt); + const [next] = orderQueuedRunsForDispatch(pending, { now, + startsSinceRecovery: lastRecovery < 0 ? recent.length : lastRecovery, + startsSinceOldest: lastOldest < 0 ? recentForOldest.length : lastOldest }); + started.push(next!); + pending = pending.filter(entry => entry.id !== next!.id); + } + const position = started.findIndex(entry => entry.id === continuation.id); + expect(position).toBeGreaterThanOrEqual(0); + expect(position).toBeLessThan(OLDEST_DISPATCH_SHARE); + expect(started.slice(0, position).every(entry => entry.wakeReason === "task_retry" || entry.wakeReason === "issue_continuation_needed")).toBe(true); + }); + + it("serves the longest-waiting run once every share even when fresher work outranks it", () => { + const people = Array.from({ length: 8 }, () => personRetry()); + const old = run({ wakeReason: "issue_monitor_due", priorityRank: 3, createdAt: minutesAgo(200) }); + const ordered = orderQueuedRunsForDispatch([...people, old], { now, startsSinceRecovery: 0, startsSinceOldest: 0 }); + expect(ordered.findIndex(entry => entry.id === old.id)).toBe(OLDEST_DISPATCH_SHARE - 1); + const due = orderQueuedRunsForDispatch([...people, old], { now, startsSinceRecovery: 0, startsSinceOldest: OLDEST_DISPATCH_SHARE - 1 }); + expect(due[0]!.id).toBe(old.id); + }); + + it("does not use the oldest-run share for a run that has not waited past the aging bound", () => { + const people = Array.from({ length: 3 }, () => personRetry()); + const recent = run({ wakeReason: "issue_assigned", createdAt: new Date(now.getTime() - QUEUED_RUN_AGING_MS + 60_000) }); + const ordered = orderQueuedRunsForDispatch([...people, recent], { now, startsSinceRecovery: 0, startsSinceOldest: OLDEST_DISPATCH_SHARE - 1 }); + expect(ordered[0]!.wakeReason).toBe("task_retry"); + }); + + it("lets an old-backlog recovery run age into the work tier", () => { + const aged = run({ wakeReason: "issue_continuation_needed", createdAt: new Date(now.getTime() - QUEUED_RUN_AGING_MS), + sourceEndedAt: minutesAgo(200) }); + const fresh = run({ wakeReason: "issue_assigned", createdAt: minutesAgo(1) }); + const ordered = orderQueuedRunsForDispatch([fresh, aged], { now, startsSinceRecovery: 0 }); + expect(ordered.map(entry => entry.id)).toEqual([aged.id, fresh.id]); + }); + + it("passes the oldest-run share to the next oldest run while the oldest one keeps failing to claim", () => { + const people = Array.from({ length: 3 }, () => personRetry()); + const stuck = run({ wakeReason: "issue_continuation_needed", createdAt: minutesAgo(300), shareDeferred: true }); + const next = run({ wakeReason: "issue_monitor_due", priorityRank: 3, createdAt: minutesAgo(200) }); + const ordered = orderQueuedRunsForDispatch([...people, stuck, next], { now, startsSinceRecovery: 0, + startsSinceOldest: OLDEST_DISPATCH_SHARE - 1 }); + expect(ordered[0]!.id).toBe(next.id); + expect(ordered.slice(1, 3).every(entry => entry.wakeReason === "task_retry")).toBe(true); + }); + + it("starts the oldest continuations of a long backlog within a bound while later assignments and fresh recovery keep arriving", () => { + const continuations = [278, 245, 243].map(age => run({ wakeReason: "issue_continuation_needed", createdAt: minutesAgo(age), + sourceEndedAt: minutesAgo(age + 2) })); + const assignments = Array.from({ length: 80 }, (_, index) => run({ wakeReason: "issue_assigned", readinessRank: 1, + createdAt: minutesAgo(230 - index * 2) })); + const started: QueuedRunDispatchFacts[] = []; + let pending: QueuedRunDispatchFacts[] = [...continuations, ...assignments]; + for (let step = 0; step < 3 * OLDEST_DISPATCH_SHARE; step += 1) { + pending.push(run({ wakeReason: "issue_recovery_action_restored", createdAt: minutesAgo(1), sourceEndedAt: minutesAgo(2) }), + run({ wakeReason: "issue_assigned", readinessRank: 1, createdAt: minutesAgo(1) })); + const recent = started.slice(-RECOVERY_DISPATCH_SHARE).reverse(); + const lastRecovery = recent.findIndex(entry => isRecoveryWakeReason(entry.wakeReason)); + const oldest = pending.reduce((left, right) => left.createdAt <= right.createdAt ? left : right); + const recentForOldest = started.slice(-OLDEST_DISPATCH_SHARE).reverse(); + const lastOldest = recentForOldest.findIndex(entry => entry.createdAt <= oldest.createdAt); + const [next] = orderQueuedRunsForDispatch(pending, { now, + startsSinceRecovery: lastRecovery < 0 ? recent.length : lastRecovery, + startsSinceOldest: lastOldest < 0 ? recentForOldest.length : lastOldest }); + started.push(next!); + pending = pending.filter(entry => entry.id !== next!.id); + } + for (const continuation of continuations) expect(started.map(entry => entry.id)).toContain(continuation.id); + }); }); diff --git a/server/src/services/heartbeat/queued-run-dispatch-order.ts b/server/src/services/heartbeat/queued-run-dispatch-order.ts index ae990639a02..6354b385c80 100644 --- a/server/src/services/heartbeat/queued-run-dispatch-order.ts +++ b/server/src/services/heartbeat/queued-run-dispatch-order.ts @@ -14,6 +14,10 @@ export const RECOVERY_BACKLOG_AGE_MS = 30 * 60_000; export const RECOVERY_DISPATCH_SHARE = 4; +export const QUEUED_RUN_AGING_MS = 60 * 60_000; + +export const OLDEST_DISPATCH_SHARE = 4; + export type QueuedRunDispatchFacts = { id: string; createdAt: Date; @@ -22,6 +26,7 @@ export type QueuedRunDispatchFacts = { sourceEndedAt: Date | null; readinessRank: number; priorityRank: number; + shareDeferred?: boolean; }; export type QueuedRunDispatchClass = "person" | "work" | "recovery" | "recovery_backlog"; @@ -46,20 +51,42 @@ export function queuedRunDispatchClass(run: QueuedRunDispatchFacts, now: Date): const CLASS_TIER: Record = { person: 0, work: 1, recovery: 1, recovery_backlog: 2 }; const NOT_READY_RANK = 3; +export type QueuedRunDispatchInput = { now: Date; startsSinceRecovery: number; startsSinceOldest?: number }; + +export function hasQueuedRunAged(run: Pick, now: Date) { + return now.getTime() - run.createdAt.getTime() >= QUEUED_RUN_AGING_MS; +} + +export function oldestReadyQueuedRun(runs: readonly T[]): T | null { + let oldest: T | null = null; + for (const run of runs) { + if (run.readinessRank >= NOT_READY_RANK || run.shareDeferred) continue; + if (!oldest || run.createdAt.getTime() < oldest.createdAt.getTime()) oldest = run; + } + return oldest; +} + export function createQueuedRunDispatcher( runs: readonly T[], - input: { now: Date; startsSinceRecovery: number }, + input: QueuedRunDispatchInput, ) { - const tier = (run: T) => run.readinessRank >= NOT_READY_RANK ? 3 : CLASS_TIER[queuedRunDispatchClass(run, input.now)]; + const tier = (run: T) => { + if (run.readinessRank >= NOT_READY_RANK) return 3; + const classTier = CLASS_TIER[queuedRunDispatchClass(run, input.now)]; + return hasQueuedRunAged(run, input.now) ? Math.min(classTier, CLASS_TIER.work) : classTier; + }; const remaining = [...runs].sort((left, right) => tier(left) - tier(right) || left.readinessRank - right.readinessRank || left.priorityRank - right.priorityRank || left.createdAt.getTime() - right.createdAt.getTime()); let sinceRecovery = input.startsSinceRecovery; + let sinceOldest = input.startsSinceOldest ?? 0; return { next(): T | null { if (!remaining.length) return null; + const oldest = sinceOldest >= OLDEST_DISPATCH_SHARE - 1 ? oldestReadyQueuedRun(remaining) : null; + if (oldest && hasQueuedRunAged(oldest, input.now)) return remaining.splice(remaining.indexOf(oldest), 1)[0]!; const recoveryIndex = sinceRecovery >= RECOVERY_DISPATCH_SHARE - 1 ? remaining.findIndex(run => run.readinessRank < NOT_READY_RANK && isSystemRecoveryRun(run)) : -1; @@ -67,13 +94,15 @@ export function createQueuedRunDispatcher( }, claimed(run: T) { sinceRecovery = isSystemRecoveryRun(run) ? 0 : sinceRecovery + 1; + const oldest = oldestReadyQueuedRun(remaining); + sinceOldest = !oldest || oldest.createdAt.getTime() >= run.createdAt.getTime() ? 0 : sinceOldest + 1; }, }; } export function orderQueuedRunsForDispatch( runs: readonly T[], - input: { now: Date; startsSinceRecovery: number }, + input: QueuedRunDispatchInput, ): T[] { const dispatcher = createQueuedRunDispatcher(runs, input); const result: T[] = [];