diff --git a/server/src/__tests__/recovery-wake-herd.test.ts b/server/src/__tests__/recovery-wake-herd.test.ts new file mode 100644 index 00000000000..117fde197b7 --- /dev/null +++ b/server/src/__tests__/recovery-wake-herd.test.ts @@ -0,0 +1,294 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + agentWakeupRequests, agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issueRecoveryActions, issues, +} from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { + RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS, RECOVERY_CONTINUATION_MAX_HOLD_MS, deliverReconciledExecutions, +} from "../services/execution-recovery-resolution.js"; +import { assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js"; +import { evaluateAssignedWork } from "../services/recovery/assigned-work-invariant.js"; +import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js"; +import { executionControlSchedule } from "../services/execution-control-work.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const suite = support.supported ? describe : describe.skip; + +suite("recovery wake herd", () => { + let db!: ReturnType; + let temporary: Awaited> | null = null; + let companyId = "", userId = ""; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-recovery-wake-herd-"); + db = createDb(temporary.connectionString); + userId = randomUUID(); + await db.insert(authUsers).values({ id: userId, name: "Board", email: `${userId}@example.test`, + emailVerified: true, createdAt: new Date(), updatedAt: new Date() }); + }, 60_000); + + afterEach(async () => { + await db.execute(sql`truncate table company_memberships, issue_recovery_actions, activity_log, agent_wakeup_requests, heartbeat_runs, issues, agents, companies cascade`); + }); + + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }); + + async function seedCompany(maxConcurrentRuns = 1) { + companyId = randomUUID(); + const agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Herd", issuePrefix: `W${companyId.replaceAll("-", "").slice(0, 6)}`, + requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: userId }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, status: "active", membershipRole: "owner" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Ponytail", role: "engineer", status: "idle", + adapterType: "paperclip_runner", adapterConfig: {}, runtimeConfig: { heartbeat: { maxConcurrentRuns } }, permissions: {} }); + return agentId; + } + + async function addAgent(maxConcurrentRuns = 1) { + const agentId = randomUUID(); + await db.insert(agents).values({ id: agentId, companyId, name: `Worker ${agentId.slice(0, 4)}`, role: "engineer", status: "idle", + adapterType: "paperclip_runner", adapterConfig: {}, runtimeConfig: { heartbeat: { maxConcurrentRuns } }, permissions: {} }); + return agentId; + } + + async function task(agentId: string, status: "todo" | "in_progress" = "in_progress") { + const [row] = await db.insert(issues).values({ companyId, title: "Continue the work", status, assigneeAgentId: agentId }).returning(); + return row!; + } + + async function sourceRun(agentId: string, issueId: string, endedMinutesAgo: number) { + const finishedAt = new Date(Date.now() - endedMinutesAgo * 60_000); + const [row] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId, invocationSource: "assignment", + status: "failed", errorCode: "server_shutdown_interrupted", startedAt: new Date(finishedAt.getTime() - 60_000), finishedAt, + createdAt: new Date(finishedAt.getTime() - 120_000), contextSnapshot: { issueId, wakeReason: "issue_assigned" } }).returning(); + return row!; + } + + async function queuedRecoveryRun(agentId: string, createdMinutesAgo: number, endedMinutesAgo = 180) { + const issue = await task(agentId); + const source = await sourceRun(agentId, issue.id, endedMinutesAgo); + const [row] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: issue.id, invocationSource: "automation", + triggerDetail: "system", status: "queued", retryOfRunId: source.id, createdAt: new Date(Date.now() - createdMinutesAgo * 60_000), + contextSnapshot: { issueId: issue.id, taskId: issue.id, wakeReason: "issue_recovery_action_restored", previousRunId: source.id, + source: "execution.reconciled", forceFreshSession: true } }).returning(); + return row!; + } + + async function queuedPersonRetry(agentId: string) { + const issue = await task(agentId); + const [request] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "on_demand", triggerDetail: "manual", + reason: "issue_commented", status: "queued", requestedByActorType: "user", requestedByActorId: userId, + payload: { issueId: issue.id } }).returning(); + const [row] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: issue.id, invocationSource: "on_demand", + triggerDetail: "manual", status: "queued", wakeupRequestId: request!.id, responsibleUserId: userId, + contextSnapshot: { issueId: issue.id, taskId: issue.id, wakeReason: "issue_commented" } }).returning(); + await db.update(agentWakeupRequests).set({ runId: row!.id }).where(eq(agentWakeupRequests.id, request!.id)); + return row!; + } + + const readRun = async (id: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!; + + it("starts a person's request ahead of a burst of recovery wakes, then runs every recovery wake", async () => { + const agentId = await seedCompany(1); + const burst: Array = []; + for (let index = 0; index < 6; index += 1) burst.push(await queuedRecoveryRun(agentId, 60 - index)); + const person = await queuedPersonRetry(agentId); + const heartbeat = heartbeatService(db); + try { + await heartbeat.resumeQueuedRuns(); + expect((await readRun(person.id)).startedAt).not.toBeNull(); + for (const run of burst) expect(await readRun(run.id)).toMatchObject({ status: "queued", startedAt: null }); + await expect.poll(async () => { + await heartbeat.resumeQueuedRuns(); + const states = await Promise.all(burst.map(run => readRun(run.id))); + return states.filter(run => run.startedAt === null).length; + }, { timeout: 60_000, interval: 250 }).toBe(0); + } finally { + await heartbeat.drainActiveRunExecutions(); + } + }, 120_000); + + async function pendingContinuation(agentId: string, endedMinutesAgo: number) { + const issue = await task(agentId, "todo"); + const source = await sourceRun(agentId, issue.id, endedMinutesAgo); + const recordedAt = new Date().toISOString(); + const [action] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issue.id, kind: "active_run_watchdog", + status: "resolved", outcome: "handed_back", resolvedAt: new Date(), ownerType: "board", returnOwnerAgentId: agentId, + cause: "legacy_execution_requires_reconciliation", fingerprint: `legacy-execution:${source.id}`, + nextAction: "Continue the task.", attemptCount: 1, + evidence: { runId: source.id, originalFailureCode: "server_shutdown_interrupted", continuationDelivery: "pending", + freshContinuation: { schema: "paperclip.fresh-continuation.v1", reason: "exact_resumption_unavailable", recordedAt }, + freshContinuationSettlement: { recordedAt }, + executionReconciliation: { runId: source.id, observedOutcome: "unknown", disposition: "continue_fresh", + actorId: "execution-recovery", recordedAt } } }).returning(); + return { issue, source, action: action! }; + } + + function fakeWake() { + return vi.fn(async (agentId: string, opts: { contextSnapshot: Record }) => { + const issueId = String(opts.contextSnapshot.issueId); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId, invocationSource: "automation", + triggerDetail: "system", status: "queued", contextSnapshot: opts.contextSnapshot }).returning(); + return run!; + }); + } + const deliveryOf = async (id: string) => + (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!.evidence.continuationDelivery; + const finishQueued = async (agentId: string) => db.update(heartbeatRuns).set({ status: "succeeded", startedAt: new Date(), finishedAt: new Date() }) + .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued"))); + + it("delivers one recovery continuation per agent at a time and drains the rest on later passes", async () => { + const agentId = await seedCompany(1); + const records: Array>> = []; + for (let index = 0; index < 4; index += 1) records.push(await pendingContinuation(agentId, 120 + index)); + const wake = fakeWake(); + + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + expect((await Promise.all(records.map(record => deliveryOf(record.action.id)))).filter(state => state === "pending")).toHaveLength(3); + expect(await executionControlSchedule(db).continuation()).not.toBeNull(); + + for (let pass = 0; pass < 10 && wake.mock.calls.length < records.length; pass += 1) { + await finishQueued(agentId); + await deliverReconciledExecutions(db, wake as never); + } + expect(wake).toHaveBeenCalledTimes(records.length); + expect(await Promise.all(records.map(record => deliveryOf(record.action.id)))).toEqual(records.map(() => "delivered")); + expect(await executionControlSchedule(db).continuation()).toBeNull(); + }, 60_000); + + it("leaves a continuation pending while the agent's queue is at its concurrency limit", async () => { + const agentId = await seedCompany(1); + const busy = await task(agentId); + await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: busy.id, invocationSource: "assignment", status: "queued", + contextSnapshot: { issueId: busy.id, wakeReason: "issue_assigned" } }); + const record = await pendingContinuation(agentId, 5); + const wake = fakeWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).not.toHaveBeenCalled(); + expect(await deliveryOf(record.action.id)).toBe("pending"); + await finishQueued(agentId); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + }, 60_000); + + it("admits a continuation held past the bound even while ordinary work keeps the queue full", async () => { + const agentId = await seedCompany(1); + const busy = await task(agentId); + await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: busy.id, invocationSource: "on_demand", status: "queued", + contextSnapshot: { issueId: busy.id, wakeReason: "task_retry" } }); + const record = await pendingContinuation(agentId, 30); + const wake = fakeWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).not.toHaveBeenCalled(); + const heldSince = new Date(Date.now() - RECOVERY_CONTINUATION_MAX_HOLD_MS - 1_000).toISOString(); + await db.update(issueRecoveryActions).set({ evidence: { ...record.action.evidence, + freshContinuation: { ...(record.action.evidence.freshContinuation as object), recordedAt: heldSince } } }) + .where(eq(issueRecoveryActions.id, record.action.id)); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + expect(await deliveryOf(record.action.id)).toBe("delivered"); + }, 60_000); + + it("does not let a fresh interruption keep an older held continuation out of a full queue", async () => { + const agentId = await seedCompany(1); + const busy = await task(agentId); + await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: busy.id, invocationSource: "on_demand", status: "queued", + contextSnapshot: { issueId: busy.id, wakeReason: "task_retry" } }); + const held = await pendingContinuation(agentId, 90); + const heldSince = new Date(Date.now() - RECOVERY_CONTINUATION_MAX_HOLD_MS - 1_000).toISOString(); + await db.update(issueRecoveryActions).set({ evidence: { ...held.action.evidence, + freshContinuation: { ...(held.action.evidence.freshContinuation as object), recordedAt: heldSince } } }) + .where(eq(issueRecoveryActions.id, held.action.id)); + const fresh = await pendingContinuation(agentId, 1); + const wake = fakeWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + expect(await deliveryOf(held.action.id)).toBe("delivered"); + expect(await deliveryOf(fresh.action.id)).toBe("pending"); + }, 60_000); + + it("admits an older continuation within a bounded number of passes while fresh interruptions keep arriving", async () => { + const agentId = await seedCompany(1); + const busy = await task(agentId); + await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: busy.id, invocationSource: "on_demand", status: "queued", + contextSnapshot: { issueId: busy.id, wakeReason: "task_retry" } }); + const start = Date.now(); + vi.useFakeTimers({ toFake: ["Date"], now: start - 1_000 }); + try { + const old = await pendingContinuation(agentId, 120); + const wake = fakeWake(); + const passInterval = 15_000; + const bound = Math.ceil(RECOVERY_CONTINUATION_MAX_HOLD_MS / passInterval) + 1; + let admittedAt = -1; + for (let pass = 0; pass <= bound && admittedAt < 0; pass += 1) { + vi.setSystemTime(start + pass * passInterval); + await pendingContinuation(agentId, 1); + await deliverReconciledExecutions(db, wake as never); + if (await deliveryOf(old.action.id) === "delivered") admittedAt = pass; + } + expect(admittedAt).toBeGreaterThan(0); + expect(admittedAt).toBeLessThanOrEqual(bound); + expect(wake).toHaveBeenCalledTimes(1); + } finally { + vi.useRealTimers(); + } + }, 120_000); + + it("does not let continuations for busy agents hide other agents' continuations", async () => { + const busyAgents = [await seedCompany(1)]; + for (let index = 1; index < 4; index += 1) busyAgents.push(await addAgent(1)); + for (const agentId of busyAgents) { + const busy = await task(agentId); + await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: busy.id, invocationSource: "assignment", status: "queued", + contextSnapshot: { issueId: busy.id, wakeReason: "issue_assigned" } }); + for (let index = 0; index < 8; index += 1) await pendingContinuation(agentId, 2 + index); + } + const idleAgent = await addAgent(1); + const idle = await pendingContinuation(idleAgent, 600); + const wake = fakeWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake.mock.calls.map(call => call[0])).toEqual([idleAgent]); + expect(await deliveryOf(idle.action.id)).toBe("delivered"); + }, 60_000); + + it("caps the continuations one pass delivers per company and delivers the freshest interruption first", async () => { + const agentIds = [await seedCompany(1)]; + for (let index = 1; index < RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS + 3; index += 1) agentIds.push(await addAgent(1)); + const old: Array>> = []; + for (const agentId of agentIds.slice(1)) old.push(await pendingContinuation(agentId, 240)); + const fresh = await pendingContinuation(agentIds[0]!, 2); + const wake = fakeWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS); + expect(wake.mock.calls[0]![0]).toBe(agentIds[0]); + expect(await deliveryOf(fresh.action.id)).toBe("delivered"); + for (let pass = 0; pass < 5; pass += 1) await deliverReconciledExecutions(db, wake as never); + expect(await Promise.all(old.map(record => deliveryOf(record.action.id)))).toEqual(old.map(() => "delivered")); + }, 60_000); + + it("reads a task whose continuation is held back as waiting for its agent, not idle and not needing Retry", async () => { + const agentId = await seedCompany(1); + const queued = await queuedRecoveryRun(agentId, 1); + const held = await pendingContinuation(agentId, 30); + const wake = fakeWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).not.toHaveBeenCalled(); + expect(await deliveryOf(held.action.id)).toBe("pending"); + expect(queued.status).toBe("queued"); + + const sweep = assignedWorkSweep(db, { enqueueWakeup: vi.fn(async () => null) }); + const [issueRow] = await db.select().from(issues).where(eq(issues.id, held.issue.id)); + const facts = (await sweep.loadFacts(companyId, [issueRow!])).get(held.issue.id)!; + expect(facts.pendingWakes).toBeGreaterThan(0); + expect(evaluateAssignedWork(facts)).toMatchObject({ kind: "ok", clause: "pending_intent", intent: "queued_wake" }); + + const presentation = await taskInterruptionPresentation(db, issueRow!); + expect(presentation).toMatchObject({ canRetry: false, state: "working", responsibility: "system" }); + }, 60_000); +}); diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index 5002ee9c15f..a14870dcbe5 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -13,9 +13,10 @@ import { isInterruptedNativePreparationCandidate } from "./cancelled-native-star import { persistActivity } from "./activity-log.js"; import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js"; import { logger } from "../middleware/logger.js"; -import { and, eq, exists, inArray, isNull, not, or, sql } from "drizzle-orm"; +import { and, asc, eq, exists, inArray, isNull, not, or, sql } from "drizzle-orm"; import { agentWakeupRequests, + agents, chatActions, environmentLeases, heartbeatRuns, @@ -39,6 +40,12 @@ import { provenGoneLegacyControllerBootId } from "./legacy-controller-lease.js"; import { continueFreshInTransaction, continueHeldExecutions, freshContinuationCheckDue, recordFreshContinuationCheck, settleCoalescedFreshContinuationHolds } from "./fresh-continuation-fallback.js"; import { alias } from "drizzle-orm/pg-core"; +import { RECOVERY_WAKE_REASONS } from "./heartbeat/queued-run-dispatch-order.js"; +import { parseHeartbeatPolicy } from "./heartbeat/timer-policy.js"; + +export const RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS = 5; +export const RECOVERY_CONTINUATION_MAX_HOLD_MS = 5 * 60_000; +const RECOVERY_CONTINUATION_PAGES_PER_PASS = 8; /** An operator records observed outcomes; this is not permission to blindly retry. */ /** @@ -357,124 +364,177 @@ export async function deliverReconciledExecutions( db: Db, wake: ReturnType["wakeup"], ) { - const pending = await db - .select() - .from(issueRecoveryActions) - .where( - and( - eq(issueRecoveryActions.status, "resolved"), - sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, - ), - ) - .limit(25); - for (const action of pending) { - try { - const decision = action.evidence.executionReconciliation as - ExecutionReconciliation | undefined; - if (!decision || !action.returnOwnerAgentId) continue; - const pendingDecision = and( - eq(issueRecoveryActions.companyId, action.companyId), - eq(issueRecoveryActions.id, action.id), - eq(issueRecoveryActions.status, "resolved"), - sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, - sql`${issueRecoveryActions.evidence}->'executionReconciliation' = ${JSON.stringify(decision)}::jsonb`, - ); - const [task] = await db - .select() - .from(issues) - .where( - and( - eq(issues.companyId, action.companyId), - eq(issues.id, action.sourceIssueId), - ), + const recoveryReasons = [...RECOVERY_WAKE_REASONS]; + const sourceRun = alias(heartbeatRuns, "continuation_source_run"); + const seenActionIds = new Set(); + const seenAgentIds = new Set(); + const admittedByCompany = new Map(); + const fullCompanies = () => [...admittedByCompany].flatMap(([companyId, count]) => + count >= RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS ? [companyId] : []); + const heldSince = sql`coalesce(${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}', ${issueRecoveryActions.resolvedAt}::text, ${issueRecoveryActions.updatedAt}::text)::timestamptz`; + const readPendingPage = async () => { + const companies = fullCompanies(); + const heldBound = new Date(Date.now() - RECOVERY_CONTINUATION_MAX_HOLD_MS).toISOString(); + const rows = await db + .select({ action: issueRecoveryActions }) + .from(issueRecoveryActions) + .leftJoin(sourceRun, and( + eq(sourceRun.companyId, issueRecoveryActions.companyId), + sql`${sourceRun.id}::text = ${issueRecoveryActions.evidence}#>>'{executionReconciliation,runId}'`, + )) + .where( + and( + eq(issueRecoveryActions.status, "resolved"), + sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, + seenActionIds.size ? not(inArray(issueRecoveryActions.id, [...seenActionIds])) : undefined, + seenAgentIds.size ? sql`coalesce(${issueRecoveryActions.returnOwnerAgentId}::text, '') <> all(${sql.param([...seenAgentIds])}::text[])` : undefined, + companies.length ? not(inArray(issueRecoveryActions.companyId, companies)) : undefined, + sql`not exists (select 1 from ${heartbeatRuns} queued_recovery + where queued_recovery.company_id = ${issueRecoveryActions.companyId} + and queued_recovery.agent_id = ${issueRecoveryActions.returnOwnerAgentId} + and queued_recovery.status = 'queued' + and queued_recovery.context_snapshot->>'wakeReason' in (${sql.join(recoveryReasons.map(reason => sql`${reason}`), sql`, `)}))`, + ), + ) + .orderBy( + sql`${heldSince} <= ${heldBound}::timestamptz desc`, + sql`case when ${heldSince} <= ${heldBound}::timestamptz then ${heldSince} end asc nulls last`, + sql`${sourceRun.finishedAt} desc nulls last`, + asc(issueRecoveryActions.updatedAt), + ) + .limit(25); + return rows.map(row => row.action); + }; + const agentHasRoom = async (action: typeof issueRecoveryActions.$inferSelect, agentId: string) => { + if ((admittedByCompany.get(action.companyId) ?? 0) >= RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS) return false; + const [agent] = await db.select().from(agents).where(and(eq(agents.companyId, action.companyId), eq(agents.id, agentId))); + if (!agent) return true; + const [queued] = await db.select({ count: sql`count(*)::int` }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, action.companyId), eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued"))); + if (Number(queued?.count ?? 0) < parseHeartbeatPolicy(agent).maxConcurrentRuns) return true; + const recordedAt = Date.parse(String((action.evidence.freshContinuation as { recordedAt?: unknown } | undefined)?.recordedAt ?? "")); + const heldSince = Number.isFinite(recordedAt) ? recordedAt : (action.resolvedAt ?? action.updatedAt).getTime(); + return Date.now() - heldSince >= RECOVERY_CONTINUATION_MAX_HOLD_MS; + }; + for (let page = 0; page < RECOVERY_CONTINUATION_PAGES_PER_PASS; page += 1) { + const pending = await readPendingPage(); + if (!pending.length) break; + for (const action of pending) { + seenActionIds.add(action.id); + if (action.returnOwnerAgentId) { + if (seenAgentIds.has(action.returnOwnerAgentId)) continue; + seenAgentIds.add(action.returnOwnerAgentId); + } + try { + const decision = action.evidence.executionReconciliation as + ExecutionReconciliation | undefined; + if (!decision || !action.returnOwnerAgentId) continue; + const pendingDecision = and( + eq(issueRecoveryActions.companyId, action.companyId), + eq(issueRecoveryActions.id, action.id), + eq(issueRecoveryActions.status, "resolved"), + sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, + sql`${issueRecoveryActions.evidence}->'executionReconciliation' = ${JSON.stringify(decision)}::jsonb`, ); - if ( - !task || - task.assigneeAgentId !== action.returnOwnerAgentId || - ["done", "cancelled"].includes(task.status) - ) { - await db - .update(issueRecoveryActions) - .set({ - evidence: sql`${issueRecoveryActions.evidence} || '{"continuationDelivery":"invalidated"}'::jsonb`, - }) - .where(pendingDecision); - await settleCoalescedFreshContinuationHolds(db, { companyId: action.companyId, primaryId: action.id, delivery: "invalidated" }); - continue; - } - // A fresh continuation is the task's next run only while nothing else - // started one: a person's Retry or saved input that ran since it was - // recorded already continues the task, so it is never a second run. - const fresh = action.evidence.freshContinuation as { recordedAt?: unknown } | undefined; - if (fresh && typeof fresh.recordedAt === "string") { - const [newer] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, action.companyId), sql`${heartbeatRuns.id}::text <> ${decision.runId}`, - or(eq(heartbeatRuns.nativeIssueId, task.id), eq(heartbeatRuns.issueId, task.id), - sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${task.id}::text`), - sql`${heartbeatRuns.createdAt} >= ${fresh.recordedAt}::timestamptz`)).limit(1); - if (newer) { - await db.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify( - { continuationDelivery: "delivered", continuationRunId: newer.id, continuationSupersededByRun: true })}::jsonb` }) - .where(pendingDecision); - await settleCoalescedFreshContinuationHolds(db, { companyId: action.companyId, primaryId: action.id, - delivery: "delivered", continuationRunId: newer.id }); - continue; - } - } - const run = await wake(action.returnOwnerAgentId, { - source: "automation", - triggerDetail: "system", - reason: "issue_recovery_action_restored", - idempotencyKey: `execution-reconciliation:${action.id}`, - payload: { issueId: task.id, recoveryActionId: action.id }, - requestedByActorType: "system", - requestedByActorId: "execution-recovery", - contextSnapshot: { - issueId: task.id, - taskId: task.id, - recoveryActionId: action.id, - previousRunId: decision.runId, - retryOfRunId: decision.runId, - forceFreshSession: true, - wakeReason: "issue_recovery_action_restored", - source: "execution.reconciled", - ...((action.evidence.freshContinuation as { isolation?: unknown } | undefined)?.isolation === "fresh_isolated_workspace" - ? { freshContinuationIsolation: action.id } : {}), - }, - }); - if (run) - await db.transaction(async (tx) => { - await tx - .update(heartbeatRuns) - .set({ retryOfRunId: decision.runId }) - .where( - and( - eq(heartbeatRuns.companyId, action.companyId), - eq(heartbeatRuns.id, run.id), - eq(heartbeatRuns.agentId, action.returnOwnerAgentId!), - sql`${heartbeatRuns.contextSnapshot}->>'recoveryActionId' = ${action.id}`, - sql`${heartbeatRuns.contextSnapshot}->>'previousRunId' = ${decision.runId}`, - ), - ); - await tx + const [task] = await db + .select() + .from(issues) + .where( + and( + eq(issues.companyId, action.companyId), + eq(issues.id, action.sourceIssueId), + ), + ); + if ( + !task || + task.assigneeAgentId !== action.returnOwnerAgentId || + ["done", "cancelled"].includes(task.status) + ) { + await db .update(issueRecoveryActions) .set({ - evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify( - { - continuationDelivery: "delivered", - continuationRunId: run.id, - }, - )}::jsonb`, + evidence: sql`${issueRecoveryActions.evidence} || '{"continuationDelivery":"invalidated"}'::jsonb`, }) .where(pendingDecision); - await settleCoalescedFreshContinuationHolds(tx as unknown as Db, { companyId: action.companyId, primaryId: action.id, - delivery: "delivered", continuationRunId: run.id }); + await settleCoalescedFreshContinuationHolds(db, { companyId: action.companyId, primaryId: action.id, delivery: "invalidated" }); + continue; + } + // A fresh continuation is the task's next run only while nothing else + // started one: a person's Retry or saved input that ran since it was + // recorded already continues the task, so it is never a second run. + const fresh = action.evidence.freshContinuation as { recordedAt?: unknown } | undefined; + if (fresh && typeof fresh.recordedAt === "string") { + const [newer] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, action.companyId), sql`${heartbeatRuns.id}::text <> ${decision.runId}`, + or(eq(heartbeatRuns.nativeIssueId, task.id), eq(heartbeatRuns.issueId, task.id), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${task.id}::text`), + sql`${heartbeatRuns.createdAt} >= ${fresh.recordedAt}::timestamptz`)).limit(1); + if (newer) { + await db.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify( + { continuationDelivery: "delivered", continuationRunId: newer.id, continuationSupersededByRun: true })}::jsonb` }) + .where(pendingDecision); + await settleCoalescedFreshContinuationHolds(db, { companyId: action.companyId, primaryId: action.id, + delivery: "delivered", continuationRunId: newer.id }); + continue; + } + } + if (!await agentHasRoom(action, action.returnOwnerAgentId)) continue; + admittedByCompany.set(action.companyId, (admittedByCompany.get(action.companyId) ?? 0) + 1); + const run = await wake(action.returnOwnerAgentId, { + source: "automation", + triggerDetail: "system", + reason: "issue_recovery_action_restored", + idempotencyKey: `execution-reconciliation:${action.id}`, + payload: { issueId: task.id, recoveryActionId: action.id }, + requestedByActorType: "system", + requestedByActorId: "execution-recovery", + contextSnapshot: { + issueId: task.id, + taskId: task.id, + recoveryActionId: action.id, + previousRunId: decision.runId, + retryOfRunId: decision.runId, + forceFreshSession: true, + wakeReason: "issue_recovery_action_restored", + source: "execution.reconciled", + ...((action.evidence.freshContinuation as { isolation?: unknown } | undefined)?.isolation === "fresh_isolated_workspace" + ? { freshContinuationIsolation: action.id } : {}), + }, }); - } catch { - logger.warn( - { recoveryActionId: action.id }, - "Reconciled execution continuation remains pending for retry", - ); + if (run) + await db.transaction(async (tx) => { + await tx + .update(heartbeatRuns) + .set({ retryOfRunId: decision.runId }) + .where( + and( + eq(heartbeatRuns.companyId, action.companyId), + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.agentId, action.returnOwnerAgentId!), + sql`${heartbeatRuns.contextSnapshot}->>'recoveryActionId' = ${action.id}`, + sql`${heartbeatRuns.contextSnapshot}->>'previousRunId' = ${decision.runId}`, + ), + ); + await tx + .update(issueRecoveryActions) + .set({ + evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify( + { + continuationDelivery: "delivered", + continuationRunId: run.id, + }, + )}::jsonb`, + }) + .where(pendingDecision); + await settleCoalescedFreshContinuationHolds(tx as unknown as Db, { companyId: action.companyId, primaryId: action.id, + delivery: "delivered", continuationRunId: run.id }); + }); + } catch { + logger.warn( + { recoveryActionId: action.id }, + "Reconciled execution continuation remains pending for retry", + ); + } } } } diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 4adbbeb259e..8f1eecf95f1 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -10,6 +10,7 @@ import { hasAutomaticInteractionDeliveryHold, INTERACTION_DELIVERY_DENIED, reche import { deferredWorkIntentWait } from "../work-intent-outbox.js"; import { notifyIssueRecovery } from "../issue-recovery-notifications.js"; import { createHeartbeatTimerPolicy } from "./timer-policy.js"; +import { RECOVERY_DISPATCH_SHARE, isRecoveryWakeReason, isSystemRecoveryRun, createQueuedRunDispatcher } from "./queued-run-dispatch-order.js"; import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { PersistedExecutionWaitRefusal, writeDeferredExecutionWait } from "../deferred-execution-wait.js"; import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js"; @@ -96,6 +97,7 @@ import { gt, gte, inArray, + isNotNull, isNull, lte, ne, @@ -2201,6 +2203,68 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende .catch(() => logger.warn({ agentId: agent.id }, "Could not attach queued agent setup connection request")); } + async function queuedRunDispatcherForAgent( + agent: typeof agents.$inferSelect, + queuedRuns: Array, + dependencyReadiness: Map, + issueById: Map, + ) { + const now = new Date(); + const wakeReasonOf = (run: typeof heartbeatRuns.$inferSelect) => + readNonEmptyString(parseObject(run.contextSnapshot).wakeReason); + const requestIds = [...new Set(queuedRuns.flatMap(run => run.wakeupRequestId ? [run.wakeupRequestId] : []))]; + const userRequests = queuedRuns.length > 1 && requestIds.length + ? await db.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, agent.companyId), inArray(agentWakeupRequests.id, requestIds), + eq(agentWakeupRequests.requestedByActorType, "user"))) + : []; + const userRequestIds = new Set(userRequests.map(row => row.id)); + const sourceRunIdOf = (run: typeof heartbeatRuns.$inferSelect) => + readNonEmptyString(parseObject(run.contextSnapshot).previousRunId) ?? run.retryOfRunId ?? null; + const recoveryRuns = queuedRuns.filter(run => isSystemRecoveryRun({ wakeReason: wakeReasonOf(run), + requestedByUser: Boolean(run.wakeupRequestId && userRequestIds.has(run.wakeupRequestId)) })); + const sourceRunIds = [...new Set(recoveryRuns.flatMap(run => sourceRunIdOf(run) ?? []))]; + const sourceRows = queuedRuns.length > 1 && sourceRunIds.length + ? await db.select({ id: heartbeatRuns.id, finishedAt: heartbeatRuns.finishedAt }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, agent.companyId), inArray(heartbeatRuns.id, sourceRunIds))) + : []; + const sourceEndedAt = new Map(sourceRows.map(row => [row.id, row.finishedAt])); + let startsSinceRecovery = 0; + if (recoveryRuns.length && recoveryRuns.length < queuedRuns.length) { + const oldestRecoveryAt = new Date(Math.min(...recoveryRuns.map(run => run.createdAt.getTime()))); + const recentStarts = await db.select({ wakeReason: sql`${heartbeatRuns.contextSnapshot}->>'wakeReason'` }) + .from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id), + isNotNull(heartbeatRuns.startedAt), gte(heartbeatRuns.startedAt, oldestRecoveryAt))) + .orderBy(desc(heartbeatRuns.startedAt)) + .limit(RECOVERY_DISPATCH_SHARE); + const lastRecovery = recentStarts.findIndex(row => isRecoveryWakeReason(row.wakeReason)); + startsSinceRecovery = lastRecovery < 0 ? recentStarts.length : lastRecovery; + } + const facts = queuedRuns.map(run => { + const issueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); + const ready = issueId ? (dependencyReadiness.get(issueId)?.isDependencyReady ?? true) : true; + const issue = issueId ? issueById.get(issueId) : null; + const sourceRunId = sourceRunIdOf(run); + return { + run, + id: run.id, + createdAt: run.createdAt, + wakeReason: wakeReasonOf(run), + requestedByUser: Boolean(run.wakeupRequestId && userRequestIds.has(run.wakeupRequestId)), + sourceEndedAt: sourceRunId ? sourceEndedAt.get(sourceRunId) ?? null : null, + readinessRank: issueId ? (ready ? (issue?.status === "in_progress" ? 0 : 1) : 3) : 2, + priorityRank: issueRunPriorityRank(issue?.priority), + }; + }); + const dispatcher = createQueuedRunDispatcher(facts, { now, startsSinceRecovery }); + const byRun = new Map(facts.map(entry => [entry.run, entry])); + return { + next: () => dispatcher.next()?.run ?? null, + claimed: (run: typeof heartbeatRuns.$inferSelect) => { const entry = byRun.get(run); if (entry) dispatcher.claimed(entry); }, + }; + } + async function startNextQueuedRunForAgent(agentId: string, attachSetupRequest = false) { // Null unless this server has not read the run-start hand-over recently. const refreshing = refreshRunAdmission(); @@ -2277,51 +2341,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende ); const issueById = new Map(issueRows.map((row) => [row.id, row])); const companyAgents = await listCompanyAgentOrgRows(agent.companyId); - const prioritizedRuns = [...queuedRuns].sort((left, right) => { - const leftIssueId = readNonEmptyString( - parseObject(left.contextSnapshot).issueId, - ); - const rightIssueId = readNonEmptyString( - parseObject(right.contextSnapshot).issueId, - ); - const leftReadiness = leftIssueId - ? dependencyReadiness.get(leftIssueId) - : null; - const rightReadiness = rightIssueId - ? dependencyReadiness.get(rightIssueId) - : null; - const leftReady = leftIssueId - ? (leftReadiness?.isDependencyReady ?? true) - : true; - const rightReady = rightIssueId - ? (rightReadiness?.isDependencyReady ?? true) - : true; - const leftIssue = leftIssueId ? issueById.get(leftIssueId) : null; - const rightIssue = rightIssueId ? issueById.get(rightIssueId) : null; - const leftRank = leftIssueId - ? leftReady - ? leftIssue?.status === "in_progress" - ? 0 - : 1 - : 3 - : 2; - const rightRank = rightIssueId - ? rightReady - ? rightIssue?.status === "in_progress" - ? 0 - : 1 - : 3 - : 2; - if (leftRank !== rightRank) return leftRank - rightRank; - const leftPriorityRank = issueRunPriorityRank(leftIssue?.priority); - const rightPriorityRank = issueRunPriorityRank(rightIssue?.priority); - if (leftPriorityRank !== rightPriorityRank) - return leftPriorityRank - rightPriorityRank; - return left.createdAt.getTime() - right.createdAt.getTime(); - }); + const dispatcher = await queuedRunDispatcherForAgent(agent, queuedRuns, dependencyReadiness, issueById); const claimedRuns: Array = []; - for (const queuedRun of prioritizedRuns) { + for (let queuedRun = dispatcher.next(); queuedRun; queuedRun = dispatcher.next()) { if (claimedRuns.length >= availableSlots || isRunAdmissionClosed()) break; let claimed: typeof heartbeatRuns.$inferSelect | null; try { @@ -2338,7 +2361,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende ); continue; } - if (claimed) claimedRuns.push(claimed); + if (claimed) { + claimedRuns.push(claimed); + dispatcher.claimed(queuedRun); + } } if (claimedRuns.length === 0) return []; diff --git a/server/src/services/heartbeat/queued-run-dispatch-order.test.ts b/server/src/services/heartbeat/queued-run-dispatch-order.test.ts new file mode 100644 index 00000000000..ae74437a1fb --- /dev/null +++ b/server/src/services/heartbeat/queued-run-dispatch-order.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from "vitest"; +import { + RECOVERY_BACKLOG_AGE_MS, + createQueuedRunDispatcher, + RECOVERY_DISPATCH_SHARE, + isRecoveryWakeReason, + orderQueuedRunsForDispatch, + queuedRunDispatchClass, + type QueuedRunDispatchFacts, +} from "./queued-run-dispatch-order.js"; + +const now = new Date("2026-11-12T23:30:00.000Z"); +const minutesAgo = (minutes: number) => new Date(now.getTime() - minutes * 60_000); + +let sequence = 0; +function run(input: Partial & { wakeReason: string }): QueuedRunDispatchFacts { + sequence += 1; + return { id: `run-${sequence}`, createdAt: minutesAgo(40), requestedByUser: false, sourceEndedAt: null, + readinessRank: 0, priorityRank: 2, ...input }; +} +const backlogRecovery = (createdMinutesAgo = 40) => run({ wakeReason: "issue_recovery_action_restored", + createdAt: minutesAgo(createdMinutesAgo), sourceEndedAt: minutesAgo(180) }); +const personRetry = (createdMinutesAgo = 1) => run({ wakeReason: "task_retry", requestedByUser: true, + createdAt: minutesAgo(createdMinutesAgo) }); + +function dispatch(queue: QueuedRunDispatchFacts[], options: { steps: number; arrivals?: (step: number) => QueuedRunDispatchFacts[] }) { + const started: QueuedRunDispatchFacts[] = []; + let pending = [...queue]; + for (let step = 0; step < options.steps && (pending.length || options.arrivals); step += 1) { + pending.push(...(options.arrivals?.(step) ?? [])); + if (!pending.length) break; + const lastRecovery = [...started].reverse().findIndex(entry => isRecoveryWakeReason(entry.wakeReason)); + const startsSinceRecovery = lastRecovery < 0 ? Math.min(started.length, RECOVERY_DISPATCH_SHARE) : lastRecovery; + const [next] = orderQueuedRunsForDispatch(pending, { now, startsSinceRecovery }); + started.push(next!); + pending = pending.filter(entry => entry.id !== next!.id); + } + return { started, pending }; +} + +describe("queued run dispatch order", () => { + it("starts a person's Retry ahead of a burst of recovery wakes, and every recovery wake still runs", () => { + const burst = Array.from({ length: 40 }, (_, index) => backlogRecovery(60 - index)); + const retry = personRetry(); + const { started, pending } = dispatch([...burst, retry], { steps: 100 }); + expect(started[0]!.id).toBe(retry.id); + expect(pending).toHaveLength(0); + expect(started.filter(entry => entry.wakeReason === "issue_recovery_action_restored")).toHaveLength(40); + }); + + it("puts chat messages and assignments by a person ahead of recovery wakes", () => { + const burst = Array.from({ length: 5 }, () => backlogRecovery()); + const chat = run({ wakeReason: "External chat message received", createdAt: minutesAgo(2) }); + const assignment = run({ wakeReason: "issue_assigned", requestedByUser: true, createdAt: minutesAgo(1) }); + const ordered = orderQueuedRunsForDispatch([...burst, chat, assignment], { now, startsSinceRecovery: 0 }); + expect(ordered.slice(0, 2).map(entry => entry.id).sort()).toEqual([chat.id, assignment.id].sort()); + }); + + it("keeps draining recovery wakes while people keep sending work", () => { + const burst = Array.from({ length: 10 }, () => backlogRecovery()); + const steps = burst.length * RECOVERY_DISPATCH_SHARE; + const { started } = dispatch(burst, { steps, arrivals: () => [personRetry()] }); + const recoveryStarts = started.filter(entry => entry.wakeReason === "issue_recovery_action_restored").length; + expect(recoveryStarts).toBe(burst.length); + for (let window = 0; window + RECOVERY_DISPATCH_SHARE <= started.length; window += 1) { + const slice = started.slice(window, window + RECOVERY_DISPATCH_SHARE); + if (window + RECOVERY_DISPATCH_SHARE <= steps) + expect(slice.some(entry => entry.wakeReason === "issue_recovery_action_restored")).toBe(true); + } + expect(started.filter(entry => entry.wakeReason === "task_retry").length).toBe(steps - burst.length); + }); + + it("keeps recovery's share when one dispatch fills many slots", () => { + const people = Array.from({ length: 12 }, () => personRetry()); + const recovery = Array.from({ length: 6 }, () => backlogRecovery()); + const batch = orderQueuedRunsForDispatch([...recovery, ...people], { now, startsSinceRecovery: 0 }).slice(0, 8); + expect(batch.map(entry => entry.wakeReason === "task_retry" ? "P" : "R").join("")).toBe("PPPRPPPR"); + }); + + it("offers the next recovery run when the reserved recovery run cannot be claimed", () => { + const people = Array.from({ length: 4 }, () => personRetry()); + const stuck = backlogRecovery(59); + const claimable = backlogRecovery(30); + const dispatcher = createQueuedRunDispatcher([...people, stuck, claimable], { now, startsSinceRecovery: RECOVERY_DISPATCH_SHARE - 1 }); + const first = dispatcher.next(); + expect(first!.id).toBe(stuck.id); + const second = dispatcher.next(); + expect(second!.id).toBe(claimable.id); + dispatcher.claimed(second!); + expect(dispatcher.next()!.wakeReason).toBe("task_retry"); + }); + + it("does not put a fresh interruption's continuation behind old-backlog recovery wakes", () => { + const backlog = Array.from({ length: 12 }, (_, index) => backlogRecovery(50 - index)); + const fresh = run({ wakeReason: "issue_recovery_action_restored", createdAt: minutesAgo(1), sourceEndedAt: minutesAgo(3) }); + expect(queuedRunDispatchClass(fresh, now)).toBe("recovery"); + expect(queuedRunDispatchClass(backlog[0]!, now)).toBe("recovery_backlog"); + const ordered = orderQueuedRunsForDispatch([...backlog, fresh], { now, startsSinceRecovery: 0 }); + expect(ordered[0]!.id).toBe(fresh.id); + }); + + it("treats a recovery wake without a known source as backlog once it has waited past the backlog age", () => { + const waited = run({ wakeReason: "issue_continuation_needed", createdAt: new Date(now.getTime() - RECOVERY_BACKLOG_AGE_MS - 1) }); + const recent = run({ wakeReason: "issue_continuation_needed", createdAt: minutesAgo(5) }); + expect(queuedRunDispatchClass(waited, now)).toBe("recovery_backlog"); + expect(queuedRunDispatchClass(recent, now)).toBe("recovery"); + }); + + it("keeps runs whose dependencies are not ready last, even a person's", () => { + const blockedRetry = run({ wakeReason: "task_retry", requestedByUser: true, readinessRank: 3, createdAt: minutesAgo(1) }); + const recovery = backlogRecovery(); + const ordered = orderQueuedRunsForDispatch([blockedRetry, recovery], { now, startsSinceRecovery: 0 }); + expect(ordered.map(entry => entry.id)).toEqual([recovery.id, blockedRetry.id]); + }); + + it("keeps the existing readiness, priority and age order within a class", () => { + const later = run({ wakeReason: "issue_assigned", createdAt: minutesAgo(1) }); + const earlier = run({ wakeReason: "issue_assigned", createdAt: minutesAgo(5) }); + const urgent = run({ wakeReason: "issue_assigned", priorityRank: 0, createdAt: minutesAgo(1) }); + const inProgress = run({ wakeReason: "issue_assigned", readinessRank: 0, priorityRank: 3, createdAt: minutesAgo(1) }); + const todo = run({ wakeReason: "issue_assigned", readinessRank: 1, priorityRank: 0, createdAt: minutesAgo(9) }); + const ordered = orderQueuedRunsForDispatch([todo, later, earlier, urgent, inProgress], { now, startsSinceRecovery: 0 }); + expect(ordered.map(entry => entry.id)).toEqual([urgent.id, earlier.id, later.id, inProgress.id, todo.id]); + }); +}); diff --git a/server/src/services/heartbeat/queued-run-dispatch-order.ts b/server/src/services/heartbeat/queued-run-dispatch-order.ts new file mode 100644 index 00000000000..ae990639a02 --- /dev/null +++ b/server/src/services/heartbeat/queued-run-dispatch-order.ts @@ -0,0 +1,85 @@ +import { EXTERNAL_CHAT_MESSAGE_WAKE_REASON } from "../queued-comment-delivery.js"; + +export const RECOVERY_WAKE_REASONS: ReadonlySet = new Set([ + "issue_recovery_action_restored", + "issue_continuation_needed", +]); + +export const PERSON_WAKE_REASONS: ReadonlySet = new Set([ + "task_retry", + EXTERNAL_CHAT_MESSAGE_WAKE_REASON, +]); + +export const RECOVERY_BACKLOG_AGE_MS = 30 * 60_000; + +export const RECOVERY_DISPATCH_SHARE = 4; + +export type QueuedRunDispatchFacts = { + id: string; + createdAt: Date; + wakeReason: string | null; + requestedByUser: boolean; + sourceEndedAt: Date | null; + readinessRank: number; + priorityRank: number; +}; + +export type QueuedRunDispatchClass = "person" | "work" | "recovery" | "recovery_backlog"; + +export function isRecoveryWakeReason(reason: string | null | undefined) { + return Boolean(reason && RECOVERY_WAKE_REASONS.has(reason)); +} + +export function isSystemRecoveryRun(run: Pick) { + return isRecoveryWakeReason(run.wakeReason) && !run.requestedByUser; +} + +export function queuedRunDispatchClass(run: QueuedRunDispatchFacts, now: Date): QueuedRunDispatchClass { + if (isSystemRecoveryRun(run)) { + const endedAt = run.sourceEndedAt ?? run.createdAt; + return now.getTime() - endedAt.getTime() > RECOVERY_BACKLOG_AGE_MS ? "recovery_backlog" : "recovery"; + } + if (run.requestedByUser || (run.wakeReason && PERSON_WAKE_REASONS.has(run.wakeReason))) return "person"; + return "work"; +} + +const CLASS_TIER: Record = { person: 0, work: 1, recovery: 1, recovery_backlog: 2 }; +const NOT_READY_RANK = 3; + +export function createQueuedRunDispatcher( + runs: readonly T[], + input: { now: Date; startsSinceRecovery: number }, +) { + const tier = (run: T) => run.readinessRank >= NOT_READY_RANK ? 3 : CLASS_TIER[queuedRunDispatchClass(run, input.now)]; + const remaining = [...runs].sort((left, right) => + tier(left) - tier(right) + || left.readinessRank - right.readinessRank + || left.priorityRank - right.priorityRank + || left.createdAt.getTime() - right.createdAt.getTime()); + let sinceRecovery = input.startsSinceRecovery; + return { + next(): T | null { + if (!remaining.length) return null; + const recoveryIndex = sinceRecovery >= RECOVERY_DISPATCH_SHARE - 1 + ? remaining.findIndex(run => run.readinessRank < NOT_READY_RANK && isSystemRecoveryRun(run)) + : -1; + return remaining.splice(recoveryIndex < 0 ? 0 : recoveryIndex, 1)[0]!; + }, + claimed(run: T) { + sinceRecovery = isSystemRecoveryRun(run) ? 0 : sinceRecovery + 1; + }, + }; +} + +export function orderQueuedRunsForDispatch( + runs: readonly T[], + input: { now: Date; startsSinceRecovery: number }, +): T[] { + const dispatcher = createQueuedRunDispatcher(runs, input); + const result: T[] = []; + for (let run = dispatcher.next(); run; run = dispatcher.next()) { + result.push(run); + dispatcher.claimed(run); + } + return result; +} diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index ebe6302bdc0..e0225712071 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -9,6 +9,7 @@ import { companies, heartbeatRuns, issueApprovals, + issueRecoveryActions, issueRelations, issueThreadInteractions, issues, @@ -276,7 +277,17 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { const companyBudget = company?.pauseReason === "budget" || company?.pauseReason === "budget_exceeded"; const counts = (rows: Array<{ issueId: string | null; count: number }>) => new Map(rows.filter(row => row.issueId).map(row => [row.issueId!, Number(row.count)])); + const pendingContinuations = await database.select({ issueId: issueRecoveryActions.sourceIssueId, + agentId: issueRecoveryActions.returnOwnerAgentId }).from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.sourceIssueId, ids), + eq(issueRecoveryActions.status, "resolved"), + sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`)); const wakeCounts = counts(wakes); + for (const continuation of pendingContinuations) { + const issue = issueRows.find(row => row.id === continuation.issueId); + if (!issue || !continuation.agentId || issue.assigneeAgentId !== continuation.agentId) continue; + wakeCounts.set(continuation.issueId, (wakeCounts.get(continuation.issueId) ?? 0) + 1); + } const chatWakeIds = new Set(chatWakes.map(row => row.id)); // A native restart-recovery claim that nobody renewed for NATIVE_STALL_MS // is not a pending path, and its still-running run is not live.