diff --git a/server/src/__tests__/assigned-work-invariant.test.ts b/server/src/__tests__/assigned-work-invariant.test.ts index 5dca17b02fe..d7c3f88b970 100644 --- a/server/src/__tests__/assigned-work-invariant.test.ts +++ b/server/src/__tests__/assigned-work-invariant.test.ts @@ -105,6 +105,30 @@ describePostgres("assigned-work invariant sweep", () => { expect(queue.calls).toHaveLength(1); }, 60_000); + it("a wake deferred behind a run that ended long ago is not a pending path; a fresh one still is", async () => { + // The 8/14 release-swap wave: a person's follow-up stayed deferred_issue_execution + // after restart recovery ended the run it waited behind, and the report said "queued_wake". + const f = await seedCompany(); + const issueId = await f.issue(); + await f.run(issueId, { status: "failed", errorCode: "native_restart_recovery_blocked" }); + const deferred = (createdAt: Date) => db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, + source: "automation", triggerDetail: "system", reason: "issue_execution_deferred", status: "deferred_issue_execution", + requestedByActorType: "user", requestedByActorId: "local-board", payload: { issueId }, createdAt, updatedAt: new Date() }); + await deferred(new Date(Date.now() - 30 * 60_000)); + const queue = wakeQueue(); + const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null }); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId })) + .toMatchObject({ outcome: "repaired", verdict: { kind: "violation", reason: "no_pending_path" } }); + + const freshIssue = await f.issue(); + await f.run(freshIssue, { status: "failed", errorCode: "native_restart_recovery_blocked" }); + await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "automation", + triggerDetail: "system", reason: "issue_execution_deferred", status: "deferred_issue_execution", + payload: { issueId: freshIssue }, createdAt: new Date(Date.now() - 60_000) }); + expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: freshIssue })) + .toMatchObject({ outcome: "ok", verdict: { clause: "pending_intent", intent: "queued_wake" } }); + }, 60_000); + it("an in_progress task whose native run failed needing reconciliation (silently skipped today) gets one continuation", async () => { const f = await seedCompany(); const issueId = await f.issue(); diff --git a/server/src/services/recovery/assigned-work-invariant.test.ts b/server/src/services/recovery/assigned-work-invariant.test.ts index 6be75f2a9d0..89e8b60119d 100644 --- a/server/src/services/recovery/assigned-work-invariant.test.ts +++ b/server/src/services/recovery/assigned-work-invariant.test.ts @@ -8,6 +8,7 @@ import { SETTLING_GRACE_MS, SETTLING_MAX_MS, STALE_BLOCK_MS, + SYSTEM_RECOVERY_WORKING_BOUND_MS, VIOLATION_REASONS, type AssignedWorkFacts, } from "./assigned-work-invariant.js"; @@ -74,7 +75,8 @@ describe("assigned-work invariant", () => { ["pending_handoff", { pendingHandoff: true }], ["session_goal", { sessionGoal: "active" }], ["scheduled_monitor", { issue: { monitorNextCheckAt: new Date(now.getTime() + 60_000) } }], - ["system_recovery_working", { interruption: { responsibility: "system", state: "working", userAction: null, nextRetryAt: null, canRetry: false } }], + ["system_recovery_working", { interruption: { responsibility: "system", state: "working", userAction: null, nextRetryAt: null, canRetry: false }, + latestRun: { id: "run-1", status: "failed", errorCode: "native_restart_recovery_blocked", finishedAt: ago(5 * 60_000), settlingUntil: null, operatorStopped: false } }], ]; for (const [intent, overrides] of cases) { it(`accepts ${intent}`, () => { @@ -82,6 +84,18 @@ describe("assigned-work invariant", () => { }); } + it("stops believing system recovery is working once the last run ended longer ago than the bound", () => { + // A recorded continuation that is never delivered keeps the projection at + // "working" forever; the invariant must see it as nothing pending. + const working = { responsibility: "system" as const, state: "working", userAction: null, nextRetryAt: null, canRetry: false }; + const run = (msAgo: number) => ({ id: "run-1", status: "failed", errorCode: "native_restart_recovery_blocked", + finishedAt: ago(msAgo), settlingUntil: null, operatorStopped: false }); + expect(evaluateAssignedWork(facts({ interruption: working, latestRun: run(SYSTEM_RECOVERY_WORKING_BOUND_MS - 60_000) }), now)) + .toEqual({ kind: "ok", clause: "pending_intent", intent: "system_recovery_working" }); + expect(evaluateAssignedWork(facts({ interruption: working, latestRun: run(SYSTEM_RECOVERY_WORKING_BOUND_MS + 60_000) }), now)) + .toMatchObject({ kind: "violation", reason: "no_pending_path" }); + }); + it("does not count a monitor or retry whose time has passed", () => { expect(evaluateAssignedWork(facts({ issue: { monitorNextCheckAt: ago(1) } }), now)).toMatchObject({ kind: "violation" }); expect(evaluateAssignedWork(facts({ interruption: { responsibility: "system", state: "retry_scheduled", userAction: null, diff --git a/server/src/services/recovery/assigned-work-invariant.ts b/server/src/services/recovery/assigned-work-invariant.ts index f9d030ea85f..204bde5c35f 100644 --- a/server/src/services/recovery/assigned-work-invariant.ts +++ b/server/src/services/recovery/assigned-work-invariant.ts @@ -100,6 +100,11 @@ const PROVIDER_LIMIT_CAUSES = new Set(["provider_quota", "native_provider_usage_ export const SETTLING_GRACE_MS = 90_000; /** Upper bound on how long another server's lease can keep a finished run settling. */ export const SETTLING_MAX_MS = 5 * 60_000; +/** "System recovery working" with nothing else pending is believed for this long + * after the task's last run ended. Every recovery wait the fallback owns ends + * within 15 minutes (saved-work recovery that keeps advancing: its own budget); + * past this the state is stale and the task is continued like any other. */ +export const SYSTEM_RECOVERY_WORKING_BOUND_MS = 20 * 60_000; /** Blocked tasks older than this are reported, never continued automatically. */ export const STALE_BLOCK_MS = 7 * 24 * 60 * 60_000; /** Automatic continuations per task, shared with the server-restart lane. */ @@ -231,7 +236,8 @@ function pendingIntent(facts: AssignedWorkFacts, now: Date): PendingIntent | nul if (facts.issue.monitorNextCheckAt && facts.issue.monitorNextCheckAt.getTime() > now.getTime()) return "scheduled_monitor"; const interruption = facts.interruption; if (interruption?.responsibility === "system") { - if (interruption.state === "working") return "system_recovery_working"; + if (interruption.state === "working" && !(facts.latestRun?.finishedAt + && now.getTime() - facts.latestRun.finishedAt.getTime() > SYSTEM_RECOVERY_WORKING_BOUND_MS)) return "system_recovery_working"; if (interruption.state === "retry_scheduled" && interruption.nextRetryAt && Date.parse(interruption.nextRetryAt) > now.getTime()) return "scheduled_retry"; } diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index e0225712071..81c05c7e38b 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -57,6 +57,8 @@ export const ASSIGNED_WORK_REPORT_LIMIT = 2_000; const OPEN_STATUSES = ["todo", "in_progress", "blocked"] as const; const PENDING_WAKE_STATUSES = ["queued", "deferred_issue_execution", "claimed"]; +/** How long a deferred wake counts as pending when no run holds its task. */ +const DEFERRED_WAKE_PENDING_BOUND_MS = 5 * 60_000; const NON_TERMINAL_RUN_STATUSES = ["queued", "running", "scheduled_retry"]; const NATIVE_PENDING_RECOVERY_STATES = ["awaiting_evidence", "awaiting_runner_reattach", "resuming_session", "bootstrap_incomplete"]; const PROVIDER_LIMIT_CAUSES = ["provider_quota", "native_provider_usage_limit"]; @@ -195,6 +197,11 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { database.select({ issueId: sql`${agentWakeupRequests.payload}->>'issueId'`, count: sql`count(*)::int` }) .from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, companyId), inArray(agentWakeupRequests.status, PENDING_WAKE_STATUSES), + // A wake deferred behind a run is promoted when that run releases the + // task. With no live run (checked first), one older than the bound has + // nothing left that will promote it, so it is not a pending path. + sql`(${agentWakeupRequests.status} <> 'deferred_issue_execution' + or ${agentWakeupRequests.createdAt} > ${at.toISOString()}::timestamptz - ${`${DEFERRED_WAKE_PENDING_BOUND_MS} milliseconds`}::interval)`, sql`${agentWakeupRequests.payload}->>'issueId' = any(${idParam})`)) .groupBy(sql`${agentWakeupRequests.payload}->>'issueId'`), database.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, ids),