From 9dd252a920f88b48697faa798df671b8db3cee27 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 12 Nov 2026 15:46:50 -0600 Subject: [PATCH] fix(recovery): let the periodic sweep find tasks behind a stale deferred wake; tie the working bound to the fallback's The suspect listing excluded every task with a deferred wake, so the 5-minute bound only applied to direct checks. 'System recovery working' is now believed for the fallback's progress bound (15 min) plus one re-check (17 min). Co-Authored-By: Claude Opus 5.5 --- server/src/__tests__/assigned-work-invariant.test.ts | 3 +++ .../src/services/recovery/assigned-work-invariant.ts | 11 +++++++---- server/src/services/recovery/assigned-work-sweep.ts | 3 ++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/server/src/__tests__/assigned-work-invariant.test.ts b/server/src/__tests__/assigned-work-invariant.test.ts index d7c3f88b970..5357c064968 100644 --- a/server/src/__tests__/assigned-work-invariant.test.ts +++ b/server/src/__tests__/assigned-work-invariant.test.ts @@ -117,6 +117,8 @@ describePostgres("assigned-work invariant sweep", () => { await deferred(new Date(Date.now() - 30 * 60_000)); const queue = wakeQueue(); const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null }); + // The periodic sweep finds it too: a stale deferred wake no longer hides the task from the suspect list. + expect((await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 50)).map(row => row.id)).toContain(issueId); expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId })) .toMatchObject({ outcome: "repaired", verdict: { kind: "violation", reason: "no_pending_path" } }); @@ -125,6 +127,7 @@ describePostgres("assigned-work invariant sweep", () => { await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "automation", triggerDetail: "system", reason: "issue_execution_deferred", status: "deferred_issue_execution", payload: { issueId: freshIssue }, createdAt: new Date(Date.now() - 60_000) }); + expect((await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 50)).map(row => row.id)).not.toContain(freshIssue); expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: freshIssue })) .toMatchObject({ outcome: "ok", verdict: { clause: "pending_intent", intent: "queued_wake" } }); }, 60_000); diff --git a/server/src/services/recovery/assigned-work-invariant.ts b/server/src/services/recovery/assigned-work-invariant.ts index 204bde5c35f..d7f4e6146ac 100644 --- a/server/src/services/recovery/assigned-work-invariant.ts +++ b/server/src/services/recovery/assigned-work-invariant.ts @@ -101,10 +101,13 @@ export const SETTLING_GRACE_MS = 90_000; /** Upper bound on how long another server's lease can keep a finished run settling. */ export const SETTLING_MAX_MS = 5 * 60_000; /** "System recovery working" with nothing else pending is believed for this long - * after the task's last run ended. Every recovery wait the fallback owns ends - * within 15 minutes (saved-work recovery that keeps advancing: its own budget); - * past this the state is stale and the task is continued like any other. */ -export const SYSTEM_RECOVERY_WORKING_BOUND_MS = 20 * 60_000; + * after the task's last run ended: the fallback's worst-case wait for exact + * recovery that shows progress (FRESH_CONTINUATION_PROGRESS_BOUND_MS, 15 min) + * plus one 2-minute disposition re-check. Past it the state is stale and the + * task is continued like any other; the sweep's wake still goes through + * admission, which hands a held task to the fallback rather than starting a + * second writer (saved-work recovery that keeps advancing keeps its budget). */ +export const SYSTEM_RECOVERY_WORKING_BOUND_MS = 17 * 60_000; /** Blocked tasks older than this are reported, never continued automatically. */ export const STALE_BLOCK_MS = 7 * 24 * 60 * 60_000; /** Automatic continuations per task, shared with the server-restart lane. */ diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index 81c05c7e38b..3da82c960a7 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -431,7 +431,8 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { and r.controller_lease_expires_at < ${at.toISOString()}::timestamptz))`, sql`not exists (select 1 from agent_wakeup_requests w where w.company_id = ${issues.companyId} and (w.payload->>'issueId') = ${issues.id}::text - and w.status in ('queued', 'deferred_issue_execution', 'claimed'))`, + and (w.status in ('queued', 'claimed') or (w.status = 'deferred_issue_execution' + and w.created_at > ${new Date(at.getTime() - DEFERRED_WAKE_PENDING_BOUND_MS).toISOString()}::timestamptz)))`, sql`not exists (select 1 from issue_thread_interactions t where t.company_id = ${issues.companyId} and t.issue_id = ${issues.id} and t.status = 'pending' and ${interactionNotSupersededSql(sql`t.company_id`, sql`t.issue_id::text`, sql`t.created_at`)})`,