mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 03:31:44 +01:00
fix(recovery): a held continuation counts as pending only within the pacing's admission bound
Paced delivery admits a held continuation within 5 minutes even when the agent's queue is full. One still pending 7 minutes after it was recorded will not be delivered, so the assigned-work sweep no longer counts it as the task's pending wake, and the invariant can repair the task. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5565476f52
commit
cfd42b0516
2 files changed
+20
-2
No files matched your search
@@ -9,7 +9,7 @@ import { heartbeatService } from "../services/heartbeat.js";
|
||||
import {
|
||||
RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS, RECOVERY_CONTINUATION_MAX_HOLD_MS, deliverReconciledExecutions,
|
||||
} from "../services/execution-recovery-resolution.js";
|
||||
import { assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js";
|
||||
import { PENDING_CONTINUATION_BOUND_MS, assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js";
|
||||
import { evaluateAssignedWork } from "../services/recovery/assigned-work-invariant.js";
|
||||
import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js";
|
||||
import { executionControlSchedule } from "../services/execution-control-work.js";
|
||||
@@ -290,5 +290,13 @@ suite("recovery wake herd", () => {
|
||||
|
||||
const presentation = await taskInterruptionPresentation(db, issueRow!);
|
||||
expect(presentation).toMatchObject({ canRetry: false, state: "working", responsibility: "system" });
|
||||
|
||||
// Pacing admits a held continuation within its hold bound. One still pending well past
|
||||
// that is not going to be delivered, so it no longer counts as the task's pending path.
|
||||
const recordedAt = new Date(Date.now() - PENDING_CONTINUATION_BOUND_MS - 60_000).toISOString();
|
||||
await db.update(issueRecoveryActions).set({ resolvedAt: new Date(recordedAt), evidence: sql`jsonb_set(${issueRecoveryActions.evidence},
|
||||
'{freshContinuation,recordedAt}', to_jsonb(${recordedAt}::text))` }).where(eq(issueRecoveryActions.id, held.action.id));
|
||||
const stale = (await sweep.loadFacts(companyId, [issueRow!])).get(held.issue.id)!;
|
||||
expect(stale.pendingWakes).toBe(facts.pendingWakes - 1);
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -59,6 +59,10 @@ const OPEN_STATUSES = ["todo", "in_progress", "blocked"] as const;
|
||||
const PENDING_WAKE_STATUSES = ["queued", "deferred_issue_execution", "claimed"];
|
||||
/** How long a deferred wake (no run holding its task) or a claimed wake with no run counts as pending. */
|
||||
const DEFERRED_WAKE_PENDING_BOUND_MS = 5 * 60_000;
|
||||
/** How long a held continuation (paced delivery) counts as pending: the pacing's
|
||||
* admission bound (RECOVERY_CONTINUATION_MAX_HOLD_MS, 5 min) plus 2 minutes of
|
||||
* 15-second delivery passes. */
|
||||
export const PENDING_CONTINUATION_BOUND_MS = 7 * 60_000;
|
||||
const NON_TERMINAL_RUN_STATUSES = ["queued", "running", "scheduled_retry"];
|
||||
const NATIVE_PENDING_RECOVERY_STATES = ["awaiting_evidence", "awaiting_runner_reattach", "resuming_session", "bootstrap_incomplete"];
|
||||
const PROVIDER_LIMIT_CAUSES = ["provider_quota", "native_provider_usage_limit"];
|
||||
@@ -293,7 +297,13 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
|
||||
agentId: issueRecoveryActions.returnOwnerAgentId }).from(issueRecoveryActions).where(and(
|
||||
eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.sourceIssueId, ids),
|
||||
eq(issueRecoveryActions.status, "resolved"),
|
||||
sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`));
|
||||
sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`,
|
||||
// Pacing admits a held continuation within RECOVERY_CONTINUATION_MAX_HOLD_MS
|
||||
// even when the agent's queue is full. One still pending past that, plus a
|
||||
// few delivery passes, is not going to be delivered: it is not a pending path.
|
||||
sql`coalesce(${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}', ${issueRecoveryActions.resolvedAt}::text,
|
||||
${issueRecoveryActions.updatedAt}::text)::timestamptz
|
||||
> ${at.toISOString()}::timestamptz - ${`${PENDING_CONTINUATION_BOUND_MS} milliseconds`}::interval`));
|
||||
const wakeCounts = counts(wakes);
|
||||
for (const continuation of pendingContinuations) {
|
||||
const issue = issueRows.find(row => row.id === continuation.issueId);
|
||||
|
||||
Reference in new issue
Block a user