fix(recovery): bound the invariant's 'pending' clauses for stale recovery state

The assigned-work invariant counted a task as pending forever when its
projection said system recovery was working, or when a wake sat deferred
behind a run that had already ended. Both were seen in the release-swap wave:
an undelivered fresh continuation and follow-ups never promoted after restart
recovery ended their run were reported OK and never repaired. 'Working' now
counts only within 20 minutes of the last run's end, and a deferred wake with
no live run only within 5 minutes of its creation; past that the task is a
violation and gets the sweep's one continuation.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-12 19:36:32 -06:00
1 parent ade7e0643f
commit b7eb506eac
4 files changed
+53 -2

No files matched your search

@@ -105,6 +105,30 @@ describePostgres("assigned-work invariant sweep", () => {
expect(queue.calls).toHaveLength(1);
}, 60_000);
it("a wake deferred behind a run that ended long ago is not a pending path; a fresh one still is", async () => {
// The 8/14 release-swap wave: a person's follow-up stayed deferred_issue_execution
// after restart recovery ended the run it waited behind, and the report said "queued_wake".
const f = await seedCompany();
const issueId = await f.issue();
await f.run(issueId, { status: "failed", errorCode: "native_restart_recovery_blocked" });
const deferred = (createdAt: Date) => db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId,
source: "automation", triggerDetail: "system", reason: "issue_execution_deferred", status: "deferred_issue_execution",
requestedByActorType: "user", requestedByActorId: "local-board", payload: { issueId }, createdAt, updatedAt: new Date() });
await deferred(new Date(Date.now() - 30 * 60_000));
const queue = wakeQueue();
const recovery = recoveryService(db, { enqueueWakeup: queue.enqueueWakeup, scheduleRecoveryRetry: async () => null });
expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId }))
.toMatchObject({ outcome: "repaired", verdict: { kind: "violation", reason: "no_pending_path" } });
const freshIssue = await f.issue();
await f.run(freshIssue, { status: "failed", errorCode: "native_restart_recovery_blocked" });
await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "automation",
triggerDetail: "system", reason: "issue_execution_deferred", status: "deferred_issue_execution",
payload: { issueId: freshIssue }, createdAt: new Date(Date.now() - 60_000) });
expect(await recovery.assignedWork.checkIssue({ companyId: f.companyId, issueId: freshIssue }))
.toMatchObject({ outcome: "ok", verdict: { clause: "pending_intent", intent: "queued_wake" } });
}, 60_000);
it("an in_progress task whose native run failed needing reconciliation (silently skipped today) gets one continuation", async () => {
const f = await seedCompany();
const issueId = await f.issue();
@@ -8,6 +8,7 @@ import {
SETTLING_GRACE_MS,
SETTLING_MAX_MS,
STALE_BLOCK_MS,
SYSTEM_RECOVERY_WORKING_BOUND_MS,
VIOLATION_REASONS,
type AssignedWorkFacts,
} from "./assigned-work-invariant.js";
@@ -74,7 +75,8 @@ describe("assigned-work invariant", () => {
["pending_handoff", { pendingHandoff: true }],
["session_goal", { sessionGoal: "active" }],
["scheduled_monitor", { issue: { monitorNextCheckAt: new Date(now.getTime() + 60_000) } }],
["system_recovery_working", { interruption: { responsibility: "system", state: "working", userAction: null, nextRetryAt: null, canRetry: false } }],
["system_recovery_working", { interruption: { responsibility: "system", state: "working", userAction: null, nextRetryAt: null, canRetry: false },
latestRun: { id: "run-1", status: "failed", errorCode: "native_restart_recovery_blocked", finishedAt: ago(5 * 60_000), settlingUntil: null, operatorStopped: false } }],
];
for (const [intent, overrides] of cases) {
it(`accepts ${intent}`, () => {
@@ -82,6 +84,18 @@ describe("assigned-work invariant", () => {
});
}
it("stops believing system recovery is working once the last run ended longer ago than the bound", () => {
// A recorded continuation that is never delivered keeps the projection at
// "working" forever; the invariant must see it as nothing pending.
const working = { responsibility: "system" as const, state: "working", userAction: null, nextRetryAt: null, canRetry: false };
const run = (msAgo: number) => ({ id: "run-1", status: "failed", errorCode: "native_restart_recovery_blocked",
finishedAt: ago(msAgo), settlingUntil: null, operatorStopped: false });
expect(evaluateAssignedWork(facts({ interruption: working, latestRun: run(SYSTEM_RECOVERY_WORKING_BOUND_MS - 60_000) }), now))
.toEqual({ kind: "ok", clause: "pending_intent", intent: "system_recovery_working" });
expect(evaluateAssignedWork(facts({ interruption: working, latestRun: run(SYSTEM_RECOVERY_WORKING_BOUND_MS + 60_000) }), now))
.toMatchObject({ kind: "violation", reason: "no_pending_path" });
});
it("does not count a monitor or retry whose time has passed", () => {
expect(evaluateAssignedWork(facts({ issue: { monitorNextCheckAt: ago(1) } }), now)).toMatchObject({ kind: "violation" });
expect(evaluateAssignedWork(facts({ interruption: { responsibility: "system", state: "retry_scheduled", userAction: null,
@@ -100,6 +100,11 @@ const PROVIDER_LIMIT_CAUSES = new Set(["provider_quota", "native_provider_usage_
export const SETTLING_GRACE_MS = 90_000;
/** Upper bound on how long another server's lease can keep a finished run settling. */
export const SETTLING_MAX_MS = 5 * 60_000;
/** "System recovery working" with nothing else pending is believed for this long
* after the task's last run ended. Every recovery wait the fallback owns ends
* within 15 minutes (saved-work recovery that keeps advancing: its own budget);
* past this the state is stale and the task is continued like any other. */
export const SYSTEM_RECOVERY_WORKING_BOUND_MS = 20 * 60_000;
/** Blocked tasks older than this are reported, never continued automatically. */
export const STALE_BLOCK_MS = 7 * 24 * 60 * 60_000;
/** Automatic continuations per task, shared with the server-restart lane. */
@@ -231,7 +236,8 @@ function pendingIntent(facts: AssignedWorkFacts, now: Date): PendingIntent | nul
if (facts.issue.monitorNextCheckAt && facts.issue.monitorNextCheckAt.getTime() > now.getTime()) return "scheduled_monitor";
const interruption = facts.interruption;
if (interruption?.responsibility === "system") {
if (interruption.state === "working") return "system_recovery_working";
if (interruption.state === "working" && !(facts.latestRun?.finishedAt
&& now.getTime() - facts.latestRun.finishedAt.getTime() > SYSTEM_RECOVERY_WORKING_BOUND_MS)) return "system_recovery_working";
if (interruption.state === "retry_scheduled" && interruption.nextRetryAt
&& Date.parse(interruption.nextRetryAt) > now.getTime()) return "scheduled_retry";
}
@@ -57,6 +57,8 @@ export const ASSIGNED_WORK_REPORT_LIMIT = 2_000;
const OPEN_STATUSES = ["todo", "in_progress", "blocked"] as const;
const PENDING_WAKE_STATUSES = ["queued", "deferred_issue_execution", "claimed"];
/** How long a deferred wake counts as pending when no run holds its task. */
const DEFERRED_WAKE_PENDING_BOUND_MS = 5 * 60_000;
const NON_TERMINAL_RUN_STATUSES = ["queued", "running", "scheduled_retry"];
const NATIVE_PENDING_RECOVERY_STATES = ["awaiting_evidence", "awaiting_runner_reattach", "resuming_session", "bootstrap_incomplete"];
const PROVIDER_LIMIT_CAUSES = ["provider_quota", "native_provider_usage_limit"];
@@ -195,6 +197,11 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
database.select({ issueId: sql<string>`${agentWakeupRequests.payload}->>'issueId'`, count: sql<number>`count(*)::int` })
.from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, companyId),
inArray(agentWakeupRequests.status, PENDING_WAKE_STATUSES),
// A wake deferred behind a run is promoted when that run releases the
// task. With no live run (checked first), one older than the bound has
// nothing left that will promote it, so it is not a pending path.
sql`(${agentWakeupRequests.status} <> 'deferred_issue_execution'
or ${agentWakeupRequests.createdAt} > ${at.toISOString()}::timestamptz - ${`${DEFERRED_WAKE_PENDING_BOUND_MS} milliseconds`}::interval)`,
sql`${agentWakeupRequests.payload}->>'issueId' = any(${idParam})`))
.groupBy(sql`${agentWakeupRequests.payload}->>'issueId'`),
database.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, ids),