diff --git a/server/src/__tests__/issue-blocker-stall-wakeups.test.ts b/server/src/__tests__/issue-blocker-stall-wakeups.test.ts index 480cc52a3d6..69f79aba045 100644 --- a/server/src/__tests__/issue-blocker-stall-wakeups.test.ts +++ b/server/src/__tests__/issue-blocker-stall-wakeups.test.ts @@ -59,6 +59,28 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu return { companyId, parentAgent, childAgent, parent, child, action }; } + it("continues a company scan past a full page of stalls, also after a restart", async () => { + const f = await fixture(); + const more = await db.insert(issues).values([1, 2].map(index => ({ companyId: f.companyId, title: `Child ${index}`, + status: "blocked", parentId: f.parent.id, assigneeAgentId: f.childAgent }))).returning(); + await db.insert(issueRecoveryActions).values(more.map(child => ({ companyId: f.companyId, sourceIssueId: child.id, + kind: "run_recovery", status: "escalated", ownerType: "board", cause: "execution_recovery_budget_exhausted", + fingerprint: `fixture-${child.id}`, nextAction: "Retry the run or reassign the task" }))); + const { calls, enqueueWakeup } = wakeRecorder(); + const stallWakes = () => new Set(calls.filter(call => call.input.reason === ISSUE_BLOCKER_STALLED_WAKE_REASON) + .map(call => call.input.idempotencyKey)); + const first = await wakeWaitersOfStalledIssues(db, enqueueWakeup as never, { companyId: f.companyId, maxStalls: 2 }); + expect(stallWakes().size).toBe(2); + expect(first.retryAt).not.toBeNull(); + vi.resetModules(); + const restarted = await import("../services/recovery/blocker-stall-wakeups.js"); + for (let pass = 0; pass < 5; pass++) { + const result = await restarted.wakeWaitersOfStalledIssues(db, enqueueWakeup as never, { companyId: f.companyId, maxStalls: 2 }); + if (result.retryAt === null) break; + } + expect(stallWakes().size).toBe(3); + }); + it("wakes the parent's assignee once per stall, with the stalled child and its recovery action", async () => { const f = await fixture(); const { calls, enqueueWakeup } = wakeRecorder(); diff --git a/server/src/__tests__/issue-recovery-deadlines.test.ts b/server/src/__tests__/issue-recovery-deadlines.test.ts index 8e3e021e852..0908b997296 100644 --- a/server/src/__tests__/issue-recovery-deadlines.test.ts +++ b/server/src/__tests__/issue-recovery-deadlines.test.ts @@ -110,6 +110,56 @@ describeIssueRecovery("issue recovery production producers and deadlines", () => expect(new Set(checked).size).toBe(1000); await quiet(); }); + it("checks only blocked tasks whose blockers are done, however many still wait", async () => { + const f = await fixture(); + const [openBlocker, doneBlocker] = await state.db.insert(issues).values([ + { companyId: f.companyId, title: "Open blocker", status: "in_progress", assigneeAgentId: f.agentId }, + { companyId: f.companyId, title: "Done blocker", status: "done" }, + ]).returning({ id: issues.id }); + const waiting = await state.db.insert(issues).values(Array.from({ length: 300 }, (_, index) => ({ + companyId: f.companyId, title: `Waiting ${index}`, status: "blocked", assigneeAgentId: f.agentId, + }))).returning({ id: issues.id }); + const [ready] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Ready", status: "blocked", + assigneeAgentId: f.agentId }).returning({ id: issues.id }); + await state.db.insert(issueRelations).values([ + ...waiting.map(row => ({ companyId: f.companyId, issueId: openBlocker.id, relatedIssueId: row.id, type: "blocks" as const })), + { companyId: f.companyId, issueId: doneBlocker.id, relatedIssueId: ready.id, type: "blocks" as const }, + ]); + const checked: string[] = []; + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + state.coordinator = createDeliveryWorkCoordinator({ owner: state.db, canRun: () => true, onError: error => state.errors.push(error) }); + const register = state.coordinator.register.bind(state.coordinator); + state.coordinator.register = (queue, task) => register(queue, { ...task, run: signal => { + const pass = task.run(signal); state.passes.add(pass); + void pass.then(() => state.passes.delete(pass), () => state.passes.delete(pass)); + return pass; + } }); + state.worker = registerIssueRecoveryWork(state.db, state.coordinator, { + stranded: async () => {}, orphanBlockers: async () => {}, recoveryActions: async () => {}, + dependencies: async scope => { checked.push(scope.issueId); return { enqueueFailed: 0 }; }, locks: async () => {}, + watchdogs: async () => ({ incomplete: 0 }), schedulingSuppressed: async () => false, + }, () => true); + await state.worker.ready; + async function settleWork() { + for (let step = 0; step < 120; step++) { + await vi.advanceTimersByTimeAsync(1000); + await Promise.allSettled([...state.passes]); await immediate(); + if (state.coordinator.nextWakeAt() === null && idleWorkSnapshot().active === 0) return; + } + throw new Error("Recovery did not settle its committed work within two virtual minutes"); + } + await settleWork(); + expect(checked).toEqual([ready.id]); + checked.length = 0; + await state.db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId })); + await settleWork(); + expect(checked).toEqual([ready.id]); + checked.length = 0; + await issueService(state.db).update(openBlocker.id, { status: "done" }); + await settleWork(); + expect(new Set(checked)).toEqual(new Set([ready.id, ...waiting.map(row => row.id)])); + await quiet(); + }); it("uses ordinary watchdog configuration and subtree closure to dispatch a reusable review", async () => { const f = await fixture(); const source = await issueService(state.db).create(f.companyId, { title: "Finished subtree", status: "done" }); diff --git a/server/src/services/recovery/blocker-stall-wakeups.ts b/server/src/services/recovery/blocker-stall-wakeups.ts index 5afc157e7e6..f8a5f5248be 100644 --- a/server/src/services/recovery/blocker-stall-wakeups.ts +++ b/server/src/services/recovery/blocker-stall-wakeups.ts @@ -288,8 +288,10 @@ export async function wakeWaitersOfStalledIssues(db: Db, enqueueWakeup: BlockerS } else { const cursorKey = opts.companyId ?? "*"; stalled = await readStalls(db, { companyId: opts.companyId, afterActionId: companyCursors.get(cursorKey) ?? null }, maxStalls); - if (stalled.length >= maxStalls) companyCursors.set(cursorKey, stalled.at(-1)!.actionId); - else companyCursors.delete(cursorKey); + if (stalled.length >= maxStalls) { + companyCursors.set(cursorKey, stalled.at(-1)!.actionId); + retryAt(Date.now()); + } else companyCursors.delete(cursorKey); } const access = authorizationService(db); const agentCanRead = async (agentId: string, onBehalfOfUserId: string | null, stall: StallRow) => (await access.decide({ diff --git a/server/src/services/recovery/work.ts b/server/src/services/recovery/work.ts index 2637f65f30f..5aa16dfc9f6 100644 --- a/server/src/services/recovery/work.ts +++ b/server/src/services/recovery/work.ts @@ -1,5 +1,5 @@ import { registerTaskWatchdogWork } from "../task-watchdog-work.js"; -import { and, asc, eq, gt, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; +import { and, asc, eq, gt, inArray, isNotNull, isNull, or, sql, type SQL } from "drizzle-orm"; import { issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js"; @@ -31,7 +31,7 @@ export function registerIssueRecoveryWork( }, canRun: () => boolean, ) { - function register(queue: DeliveryQueue, statuses: string[] | null, check: (scope: Scope) => Promise) { + function register(queue: DeliveryQueue, statuses: string[] | null, check: (scope: Scope) => Promise, eligible?: SQL) { const cursors = new Map(); let enabled = false; return registerScopedDeadlineWork(coordinator, { @@ -52,6 +52,7 @@ export function registerIssueRecoveryWork( statuses ? isNull(issues.hiddenAt) : undefined, statuses ? isNull(issues.assigneeUserId) : undefined, statuses ? or(isNotNull(issues.assigneeAgentId), eq(issues.status, "in_review")) : undefined, + eligible, )).orderBy(asc(issues.id)).limit(PAGE_SIZE); const complete = rows.length < PAGE_SIZE; if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); @@ -154,7 +155,12 @@ export function registerIssueRecoveryWork( const dependencies = register(DELIVERY_QUEUES.resolvedDependencies, ["blocked"], async scope => { const result = await owner.dependencies(scope); if (result.enqueueFailed > 0) throw new Error("Resolved dependency wake remains undelivered"); - }); + }, sql`exists (select 1 from ${issueRelations} r where r.company_id = ${issues.companyId} + and r.related_issue_id = ${issues.id} and r.type = 'blocks') + and not exists (select 1 from ${issueRelations} r inner join ${issues} blocker + on blocker.id = r.issue_id and blocker.company_id = r.company_id + where r.company_id = ${issues.companyId} and r.related_issue_id = ${issues.id} + and r.type = 'blocks' and blocker.status <> 'done')`); const locks = register(DELIVERY_QUEUES.staleIssueLocks, null, owner.locks); const assignedWork = owner.assignedWork ? registerAssignedWork(owner.assignedWork) : null; const watchdogs = registerTaskWatchdogWork(db, coordinator, {