fix(recovery): let the periodic sweep find tasks behind a stale deferred wake; tie the working bound to the fallback's

The suspect listing excluded every task with a deferred wake, so the 5-minute
bound only applied to direct checks. 'System recovery working' is now believed
for the fallback's progress bound (15 min) plus one re-check (17 min).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-12 19:36:32 -06:00
1 parent b7eb506eac
commit 9dd252a920
3 files changed
+12 -5

No files matched your search

@@ -117,6 +117,8 @@ describePostgres("assigned-work invariant sweep", () => {
await deferred(new Date(Date.now() - 30 * 60_000));
const queue = wakeQueue();
const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null });
// The periodic sweep finds it too: a stale deferred wake no longer hides the task from the suspect list.
expect((await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 50)).map(row => row.id)).toContain(issueId);
expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId }))
.toMatchObject({ outcome: "repaired", verdict: { kind: "violation", reason: "no_pending_path" } });
@@ -125,6 +127,7 @@ describePostgres("assigned-work invariant sweep", () => {
await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "automation",
triggerDetail: "system", reason: "issue_execution_deferred", status: "deferred_issue_execution",
payload: { issueId: freshIssue }, createdAt: new Date(Date.now() - 60_000) });
expect((await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 50)).map(row => row.id)).not.toContain(freshIssue);
expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: freshIssue }))
.toMatchObject({ outcome: "ok", verdict: { clause: "pending_intent", intent: "queued_wake" } });
}, 60_000);
@@ -101,10 +101,13 @@ export const SETTLING_GRACE_MS = 90_000;
/** Upper bound on how long another server's lease can keep a finished run settling. */
export const SETTLING_MAX_MS = 5 * 60_000;
/** "System recovery working" with nothing else pending is believed for this long
* after the task's last run ended. Every recovery wait the fallback owns ends
* within 15 minutes (saved-work recovery that keeps advancing: its own budget);
* past this the state is stale and the task is continued like any other. */
export const SYSTEM_RECOVERY_WORKING_BOUND_MS = 20 * 60_000;
* after the task's last run ended: the fallback's worst-case wait for exact
* recovery that shows progress (FRESH_CONTINUATION_PROGRESS_BOUND_MS, 15 min)
* plus one 2-minute disposition re-check. Past it the state is stale and the
* task is continued like any other; the sweep's wake still goes through
* admission, which hands a held task to the fallback rather than starting a
* second writer (saved-work recovery that keeps advancing keeps its budget). */
export const SYSTEM_RECOVERY_WORKING_BOUND_MS = 17 * 60_000;
/** Blocked tasks older than this are reported, never continued automatically. */
export const STALE_BLOCK_MS = 7 * 24 * 60 * 60_000;
/** Automatic continuations per task, shared with the server-restart lane. */
@@ -431,7 +431,8 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
and r.controller_lease_expires_at < ${at.toISOString()}::timestamptz))`,
sql`not exists (select 1 from agent_wakeup_requests w where w.company_id = ${issues.companyId}
and (w.payload->>'issueId') = ${issues.id}::text
and w.status in ('queued', 'deferred_issue_execution', 'claimed'))`,
and (w.status in ('queued', 'claimed') or (w.status = 'deferred_issue_execution'
and w.created_at > ${new Date(at.getTime() - DEFERRED_WAKE_PENDING_BOUND_MS).toISOString()}::timestamptz)))`,
sql`not exists (select 1 from issue_thread_interactions t where t.company_id = ${issues.companyId}
and t.issue_id = ${issues.id} and t.status = 'pending'
and ${interactionNotSupersededSql(sql`t.company_id`, sql`t.issue_id::text`, sql`t.created_at`)})`,