mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-22 11:54:35 +01:00
fix(recovery): let the periodic sweep find tasks behind a stale deferred wake; tie the working bound to the fallback's
The suspect listing excluded every task with a deferred wake, so the 5-minute bound only applied to direct checks. 'System recovery working' is now believed for the fallback's progress bound (15 min) plus one re-check (17 min). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b7eb506eac
commit
9dd252a920
3 files changed
+12
-5
No files matched your search
@@ -117,6 +117,8 @@ describePostgres("assigned-work invariant sweep", () => {
|
||||
await deferred(new Date(Date.now() - 30 * 60_000));
|
||||
const queue = wakeQueue();
|
||||
const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null });
|
||||
// The periodic sweep finds it too: a stale deferred wake no longer hides the task from the suspect list.
|
||||
expect((await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 50)).map(row => row.id)).toContain(issueId);
|
||||
expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId }))
|
||||
.toMatchObject({ outcome: "repaired", verdict: { kind: "violation", reason: "no_pending_path" } });
|
||||
|
||||
@@ -125,6 +127,7 @@ describePostgres("assigned-work invariant sweep", () => {
|
||||
await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "automation",
|
||||
triggerDetail: "system", reason: "issue_execution_deferred", status: "deferred_issue_execution",
|
||||
payload: { issueId: freshIssue }, createdAt: new Date(Date.now() - 60_000) });
|
||||
expect((await recovery.assignedWork.listSuspects({ companyId: f.companyId }, null, 50)).map(row => row.id)).not.toContain(freshIssue);
|
||||
expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: freshIssue }))
|
||||
.toMatchObject({ outcome: "ok", verdict: { clause: "pending_intent", intent: "queued_wake" } });
|
||||
}, 60_000);
|
||||
|
||||
@@ -101,10 +101,13 @@ export const SETTLING_GRACE_MS = 90_000;
|
||||
/** Upper bound on how long another server's lease can keep a finished run settling. */
|
||||
export const SETTLING_MAX_MS = 5 * 60_000;
|
||||
/** "System recovery working" with nothing else pending is believed for this long
|
||||
* after the task's last run ended. Every recovery wait the fallback owns ends
|
||||
* within 15 minutes (saved-work recovery that keeps advancing: its own budget);
|
||||
* past this the state is stale and the task is continued like any other. */
|
||||
export const SYSTEM_RECOVERY_WORKING_BOUND_MS = 20 * 60_000;
|
||||
* after the task's last run ended: the fallback's worst-case wait for exact
|
||||
* recovery that shows progress (FRESH_CONTINUATION_PROGRESS_BOUND_MS, 15 min)
|
||||
* plus one 2-minute disposition re-check. Past it the state is stale and the
|
||||
* task is continued like any other; the sweep's wake still goes through
|
||||
* admission, which hands a held task to the fallback rather than starting a
|
||||
* second writer (saved-work recovery that keeps advancing keeps its budget). */
|
||||
export const SYSTEM_RECOVERY_WORKING_BOUND_MS = 17 * 60_000;
|
||||
/** Blocked tasks older than this are reported, never continued automatically. */
|
||||
export const STALE_BLOCK_MS = 7 * 24 * 60 * 60_000;
|
||||
/** Automatic continuations per task, shared with the server-restart lane. */
|
||||
|
||||
@@ -431,7 +431,8 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
|
||||
and r.controller_lease_expires_at < ${at.toISOString()}::timestamptz))`,
|
||||
sql`not exists (select 1 from agent_wakeup_requests w where w.company_id = ${issues.companyId}
|
||||
and (w.payload->>'issueId') = ${issues.id}::text
|
||||
and w.status in ('queued', 'deferred_issue_execution', 'claimed'))`,
|
||||
and (w.status in ('queued', 'claimed') or (w.status = 'deferred_issue_execution'
|
||||
and w.created_at > ${new Date(at.getTime() - DEFERRED_WAKE_PENDING_BOUND_MS).toISOString()}::timestamptz)))`,
|
||||
sql`not exists (select 1 from issue_thread_interactions t where t.company_id = ${issues.companyId}
|
||||
and t.issue_id = ${issues.id} and t.status = 'pending'
|
||||
and ${interactionNotSupersededSql(sql`t.company_id`, sql`t.issue_id::text`, sql`t.created_at`)})`,
|
||||
|
||||
Reference in new issue
Block a user