fix(recovery): a held continuation counts as pending only within the pacing's admission bound

Paced delivery admits a held continuation within 5 minutes even when the
agent's queue is full. One still pending 7 minutes after it was recorded will
not be delivered, so the assigned-work sweep no longer counts it as the task's
pending wake, and the invariant can repair the task.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-12 19:38:22 -06:00
1 parent 5565476f52
commit cfd42b0516
2 files changed
+20 -2

No files matched your search

@@ -9,7 +9,7 @@ import { heartbeatService } from "../services/heartbeat.js";
import {
RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS, RECOVERY_CONTINUATION_MAX_HOLD_MS, deliverReconciledExecutions,
} from "../services/execution-recovery-resolution.js";
import { assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js";
import { PENDING_CONTINUATION_BOUND_MS, assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js";
import { evaluateAssignedWork } from "../services/recovery/assigned-work-invariant.js";
import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js";
import { executionControlSchedule } from "../services/execution-control-work.js";
@@ -290,5 +290,13 @@ suite("recovery wake herd", () => {
const presentation = await taskInterruptionPresentation(db, issueRow!);
expect(presentation).toMatchObject({ canRetry: false, state: "working", responsibility: "system" });
// Pacing admits a held continuation within its hold bound. One still pending well past
// that is not going to be delivered, so it no longer counts as the task's pending path.
const recordedAt = new Date(Date.now() - PENDING_CONTINUATION_BOUND_MS - 60_000).toISOString();
await db.update(issueRecoveryActions).set({ resolvedAt: new Date(recordedAt), evidence: sql`jsonb_set(${issueRecoveryActions.evidence},
'{freshContinuation,recordedAt}', to_jsonb(${recordedAt}::text))` }).where(eq(issueRecoveryActions.id, held.action.id));
const stale = (await sweep.loadFacts(companyId, [issueRow!])).get(held.issue.id)!;
expect(stale.pendingWakes).toBe(facts.pendingWakes - 1);
}, 60_000);
});
@@ -59,6 +59,10 @@ const OPEN_STATUSES = ["todo", "in_progress", "blocked"] as const;
const PENDING_WAKE_STATUSES = ["queued", "deferred_issue_execution", "claimed"];
/** How long a deferred wake (no run holding its task) or a claimed wake with no run counts as pending. */
const DEFERRED_WAKE_PENDING_BOUND_MS = 5 * 60_000;
/** How long a held continuation (paced delivery) counts as pending: the pacing's
* admission bound (RECOVERY_CONTINUATION_MAX_HOLD_MS, 5 min) plus 2 minutes of
* 15-second delivery passes. */
export const PENDING_CONTINUATION_BOUND_MS = 7 * 60_000;
const NON_TERMINAL_RUN_STATUSES = ["queued", "running", "scheduled_retry"];
const NATIVE_PENDING_RECOVERY_STATES = ["awaiting_evidence", "awaiting_runner_reattach", "resuming_session", "bootstrap_incomplete"];
const PROVIDER_LIMIT_CAUSES = ["provider_quota", "native_provider_usage_limit"];
@@ -293,7 +297,13 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
agentId: issueRecoveryActions.returnOwnerAgentId }).from(issueRecoveryActions).where(and(
eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.sourceIssueId, ids),
eq(issueRecoveryActions.status, "resolved"),
sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`));
sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`,
// Pacing admits a held continuation within RECOVERY_CONTINUATION_MAX_HOLD_MS
// even when the agent's queue is full. One still pending past that, plus a
// few delivery passes, is not going to be delivered: it is not a pending path.
sql`coalesce(${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}', ${issueRecoveryActions.resolvedAt}::text,
${issueRecoveryActions.updatedAt}::text)::timestamptz
> ${at.toISOString()}::timestamptz - ${`${PENDING_CONTINUATION_BOUND_MS} milliseconds`}::interval`));
const wakeCounts = counts(wakes);
for (const continuation of pendingContinuations) {
const issue = issueRows.find(row => row.id === continuation.issueId);