diff --git a/server/src/__tests__/recovery-wake-herd.test.ts b/server/src/__tests__/recovery-wake-herd.test.ts index 07c27485ebe..8335e518433 100644 --- a/server/src/__tests__/recovery-wake-herd.test.ts +++ b/server/src/__tests__/recovery-wake-herd.test.ts @@ -7,7 +7,8 @@ import { import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { heartbeatService } from "../services/heartbeat.js"; import { - RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS, RECOVERY_CONTINUATION_MAX_HOLD_MS, deliverReconciledExecutions, + RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS, RECOVERY_CONTINUATION_DELIVERY_BOUND_MS, RECOVERY_CONTINUATION_MAX_HOLD_MS, + deliverReconciledExecutions, } from "../services/execution-recovery-resolution.js"; import { assignedWorkSweep } from "../services/recovery/assigned-work-sweep.js"; import { evaluateAssignedWork } from "../services/recovery/assigned-work-invariant.js"; @@ -318,6 +319,135 @@ suite("recovery wake herd", () => { expect(await Promise.all(old.map(record => deliveryOf(record.action.id)))).toEqual(old.map(() => "delivered")); }, 60_000); + async function holdPastBound(record: Awaited>, extraMs = 1_000) { + const heldSince = new Date(Date.now() - RECOVERY_CONTINUATION_MAX_HOLD_MS - extraMs).toISOString(); + await db.update(issueRecoveryActions).set({ evidence: { ...record.action.evidence, + freshContinuation: { ...(record.action.evidence.freshContinuation as object), recordedAt: heldSince } } }) + .where(eq(issueRecoveryActions.id, record.action.id)); + } + + it("delivers a continuation held past the bound while the agent's queue is full of recovery runs that never start", async () => { + const agentId = await seedCompany(1); + await queuedRecoveryRun(agentId, 240); + await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: (await task(agentId)).id, invocationSource: "automation", + triggerDetail: "system", status: "queued", createdAt: new Date(Date.now() - 230 * 60_000), + contextSnapshot: { wakeReason: "issue_continuation_needed" } }); + const record = await pendingContinuation(agentId, 200); + const wake = fakeWake(); + await deliverReconciledExecutions(db, wake as never); + expect(wake).not.toHaveBeenCalled(); + expect(await deliveryOf(record.action.id)).toBe("pending"); + + await holdPastBound(record); + await deliverReconciledExecutions(db, wake as never); + expect(wake).toHaveBeenCalledTimes(1); + expect(wake.mock.calls[0]![0]).toBe(agentId); + expect(await deliveryOf(record.action.id)).toBe("delivered"); + }, 60_000); + + it("delivers the continuation after a restart while it was pending past the bound with a full queue", async () => { + const agentId = await seedCompany(1); + await queuedRecoveryRun(agentId, 240); + const record = await pendingContinuation(agentId, 200); + await deliverReconciledExecutions(db, fakeWake() as never); + expect(await deliveryOf(record.action.id)).toBe("pending"); + await holdPastBound(record, 3 * 60 * 60_000); + + const restartedSchedule = executionControlSchedule(db); + const due = await restartedSchedule.continuation(); + expect(due).not.toBeNull(); + expect(due!).toBeLessThanOrEqual(Date.now() + 60_000); + const restartedWake = fakeWake(); + await deliverReconciledExecutions(db, restartedWake as never); + expect(restartedWake).toHaveBeenCalledTimes(1); + expect(await deliveryOf(record.action.id)).toBe("delivered"); + expect(await restartedSchedule.continuation()).toBeNull(); + }, 60_000); + + it("does not let a continuation whose wake cannot enter the queue hold back other continuations", async () => { + const agentId = await seedCompany(1); + const stuck: Array>> = []; + for (let index = 0; index < RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS; index += 1) { + const record = await pendingContinuation(await addAgent(1), 300); + await holdPastBound(record, 60 * 60_000); + stuck.push(record); + } + const sameAgentStuck = await pendingContinuation(agentId, 300); + await holdPastBound(sameAgentStuck, 60 * 60_000); + const waiting = await pendingContinuation(agentId, 10); + await holdPastBound(waiting); + const stuckIds = new Set([...stuck, sameAgentStuck].map(record => record.action.id)); + const created = fakeWake(); + const wake = vi.fn(async (wakeAgentId: string, opts: { contextSnapshot: Record }) => + stuckIds.has(String(opts.contextSnapshot.recoveryActionId)) ? null : created(wakeAgentId, opts)); + await deliverReconciledExecutions(db, wake as never); + expect(created).not.toHaveBeenCalled(); + expect(await deliveryOf(waiting.action.id)).toBe("pending"); + await deliverReconciledExecutions(db, wake as never); + expect(created).toHaveBeenCalledTimes(1); + expect(await deliveryOf(waiting.action.id)).toBe("delivered"); + }, 60_000); + + it("delivers an overdue continuation after a temporary refusal while new continuations keep arriving for a full queue", async () => { + const agentId = await seedCompany(1); + const busy = await task(agentId); + await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: busy.id, invocationSource: "on_demand", status: "queued", + contextSnapshot: { issueId: busy.id, wakeReason: "task_retry" } }); + const overdue = await pendingContinuation(agentId, 240); + await holdPastBound(overdue); + const created = fakeWake(); + let refusing = true; + const wake = vi.fn(async (wakeAgentId: string, opts: { contextSnapshot: Record }) => + refusing ? null : created(wakeAgentId, opts)); + await deliverReconciledExecutions(db, wake as never); + expect(await deliveryOf(overdue.action.id)).toBe("pending"); + refusing = false; + for (let pass = 0; pass < 3 && await deliveryOf(overdue.action.id) === "pending"; pass += 1) { + await pendingContinuation(agentId, 1); + await deliverReconciledExecutions(db, wake as never); + } + expect(await deliveryOf(overdue.action.id)).toBe("delivered"); + expect(created.mock.calls[0]![1].contextSnapshot.recoveryActionId).toBe(overdue.action.id); + }, 60_000); + + it("delivers another agent's continuation in one pass while one agent refuses a backlog of more than 200", async () => { + const refusingAgent = await seedCompany(1); + const heldSince = new Date(Date.now() - 3 * 60 * 60_000).toISOString(); + const backlog = await db.insert(issues).values(Array.from({ length: 210 }, () => ({ companyId, title: "Continue the work", + status: "todo" as const, assigneeAgentId: refusingAgent }))).returning(); + await db.insert(issueRecoveryActions).values(backlog.map(issue => { + const runId = randomUUID(); + return { companyId, sourceIssueId: issue.id, kind: "active_run_watchdog" as const, status: "resolved" as const, + outcome: "handed_back" as const, resolvedAt: new Date(heldSince), ownerType: "board" as const, returnOwnerAgentId: refusingAgent, + cause: "legacy_execution_requires_reconciliation", fingerprint: `legacy-execution:${runId}`, nextAction: "Continue the task.", + attemptCount: 1, evidence: { runId, continuationDelivery: "pending", + freshContinuation: { schema: "paperclip.fresh-continuation.v1", reason: "exact_resumption_unavailable", recordedAt: heldSince }, + executionReconciliation: { runId, observedOutcome: "unknown", disposition: "continue_fresh", + actorId: "execution-recovery", recordedAt: heldSince } } }; + })); + const healthy = await pendingContinuation(await addAgent(1), 30); + const created = fakeWake(); + const wake = vi.fn(async (wakeAgentId: string, opts: { contextSnapshot: Record }) => + wakeAgentId === refusingAgent ? null : created(wakeAgentId, opts)); + await deliverReconciledExecutions(db, wake as never); + expect(wake.mock.calls.filter(call => call[0] === refusingAgent)).toHaveLength(1); + expect(created).toHaveBeenCalledTimes(1); + expect(await deliveryOf(healthy.action.id)).toBe("delivered"); + }, 120_000); + + it("does not show a task as working when its continuation is pending past the delivery bound and nothing runs", async () => { + const agentId = await seedCompany(1); + const record = await pendingContinuation(agentId, 240); + await holdPastBound(record, RECOVERY_CONTINUATION_DELIVERY_BOUND_MS); + const [issueRow] = await db.select().from(issues).where(eq(issues.id, record.issue.id)); + const presentation = await taskInterruptionPresentation(db, issueRow!); + expect(presentation?.state).not.toBe("working"); + + const fresh = await pendingContinuation(agentId, 30); + const [freshRow] = await db.select().from(issues).where(eq(issues.id, fresh.issue.id)); + expect(await taskInterruptionPresentation(db, freshRow!)).toMatchObject({ state: "working", canRetry: false }); + }, 60_000); + it("reads a task whose continuation is held back as waiting for its agent, not idle and not needing Retry", async () => { const agentId = await seedCompany(1); const queued = await queuedRecoveryRun(agentId, 1); diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index a14870dcbe5..caa6016fa73 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -43,8 +43,11 @@ import { alias } from "drizzle-orm/pg-core"; import { RECOVERY_WAKE_REASONS } from "./heartbeat/queued-run-dispatch-order.js"; import { parseHeartbeatPolicy } from "./heartbeat/timer-policy.js"; -export const RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS = 5; -export const RECOVERY_CONTINUATION_MAX_HOLD_MS = 5 * 60_000; +import { RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS, RECOVERY_CONTINUATION_MAX_HOLD_MS, + recoveryContinuationHeldSince } from "./recovery-continuation-pacing.js"; + +export { RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS, RECOVERY_CONTINUATION_DELIVERY_BOUND_MS, + RECOVERY_CONTINUATION_MAX_HOLD_MS } from "./recovery-continuation-pacing.js"; const RECOVERY_CONTINUATION_PAGES_PER_PASS = 8; /** An operator records observed outcomes; this is not permission to blindly retry. */ @@ -375,8 +378,27 @@ export async function deliverReconciledExecutions( const readPendingPage = async () => { const companies = fullCompanies(); const heldBound = new Date(Date.now() - RECOVERY_CONTINUATION_MAX_HOLD_MS).toISOString(); - const rows = await db - .select({ action: issueRecoveryActions }) + const pastBound = sql`${heldSince} <= ${heldBound}::timestamptz`; + const overdueTurn = sql`case when ${pastBound} then greatest(${heldSince}, + (${issueRecoveryActions.evidence}->>'continuationDeliveryAttemptedAt')::timestamptz) end`; + const priority = [ + sql`${pastBound} desc`, + sql`${overdueTurn} asc nulls last`, + sql`${sourceRun.finishedAt} desc nulls last`, + sql`${issueRecoveryActions.updatedAt} asc`, + ]; + const candidates = db + .select({ + id: issueRecoveryActions.id, + pastBound: sql`${pastBound}`.as("past_bound"), + heldAt: sql`${overdueTurn}`.as("held_at"), + sourceFinishedAt: sql`${sourceRun.finishedAt}`.as("source_finished_at"), + actionUpdatedAt: sql`${issueRecoveryActions.updatedAt}`.as("action_updated_at"), + agentRank: sql`row_number() over ( + partition by coalesce(${issueRecoveryActions.returnOwnerAgentId}::text, ${issueRecoveryActions.id}::text) + order by ${sql.join(priority, sql`, `)})` + .as("agent_rank"), + }) .from(issueRecoveryActions) .leftJoin(sourceRun, and( eq(sourceRun.companyId, issueRecoveryActions.companyId), @@ -389,22 +411,37 @@ export async function deliverReconciledExecutions( seenActionIds.size ? not(inArray(issueRecoveryActions.id, [...seenActionIds])) : undefined, seenAgentIds.size ? sql`coalesce(${issueRecoveryActions.returnOwnerAgentId}::text, '') <> all(${sql.param([...seenAgentIds])}::text[])` : undefined, companies.length ? not(inArray(issueRecoveryActions.companyId, companies)) : undefined, - sql`not exists (select 1 from ${heartbeatRuns} queued_recovery + sql`(${pastBound} or not exists (select 1 from ${heartbeatRuns} queued_recovery where queued_recovery.company_id = ${issueRecoveryActions.companyId} and queued_recovery.agent_id = ${issueRecoveryActions.returnOwnerAgentId} and queued_recovery.status = 'queued' - and queued_recovery.context_snapshot->>'wakeReason' in (${sql.join(recoveryReasons.map(reason => sql`${reason}`), sql`, `)}))`, + and queued_recovery.context_snapshot->>'wakeReason' in (${sql.join(recoveryReasons.map(reason => sql`${reason}`), sql`, `)})))`, ), ) + .as("continuation_candidates"); + const rows = await db + .select({ action: issueRecoveryActions }) + .from(candidates) + .innerJoin(issueRecoveryActions, eq(issueRecoveryActions.id, candidates.id)) + .where(eq(candidates.agentRank, 1)) .orderBy( - sql`${heldSince} <= ${heldBound}::timestamptz desc`, - sql`case when ${heldSince} <= ${heldBound}::timestamptz then ${heldSince} end asc nulls last`, - sql`${sourceRun.finishedAt} desc nulls last`, - asc(issueRecoveryActions.updatedAt), + sql`${candidates.pastBound} desc`, + sql`${candidates.heldAt} asc nulls last`, + sql`${candidates.sourceFinishedAt} desc nulls last`, + sql`${candidates.actionUpdatedAt} asc`, ) .limit(25); return rows.map(row => row.action); }; + const deferRefusedContinuation = (action: typeof issueRecoveryActions.$inferSelect) => db.update(issueRecoveryActions) + .set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify( + { continuationDeliveryAttemptedAt: new Date().toISOString() })}::jsonb` }) + .where(and( + eq(issueRecoveryActions.companyId, action.companyId), + eq(issueRecoveryActions.id, action.id), + eq(issueRecoveryActions.status, "resolved"), + sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, + )); const agentHasRoom = async (action: typeof issueRecoveryActions.$inferSelect, agentId: string) => { if ((admittedByCompany.get(action.companyId) ?? 0) >= RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS) return false; const [agent] = await db.select().from(agents).where(and(eq(agents.companyId, action.companyId), eq(agents.id, agentId))); @@ -412,19 +449,14 @@ export async function deliverReconciledExecutions( const [queued] = await db.select({ count: sql`count(*)::int` }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, action.companyId), eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued"))); if (Number(queued?.count ?? 0) < parseHeartbeatPolicy(agent).maxConcurrentRuns) return true; - const recordedAt = Date.parse(String((action.evidence.freshContinuation as { recordedAt?: unknown } | undefined)?.recordedAt ?? "")); - const heldSince = Number.isFinite(recordedAt) ? recordedAt : (action.resolvedAt ?? action.updatedAt).getTime(); - return Date.now() - heldSince >= RECOVERY_CONTINUATION_MAX_HOLD_MS; + return Date.now() - recoveryContinuationHeldSince(action) >= RECOVERY_CONTINUATION_MAX_HOLD_MS; }; for (let page = 0; page < RECOVERY_CONTINUATION_PAGES_PER_PASS; page += 1) { const pending = await readPendingPage(); if (!pending.length) break; for (const action of pending) { seenActionIds.add(action.id); - if (action.returnOwnerAgentId) { - if (seenAgentIds.has(action.returnOwnerAgentId)) continue; - seenAgentIds.add(action.returnOwnerAgentId); - } + if (action.returnOwnerAgentId && seenAgentIds.has(action.returnOwnerAgentId)) continue; try { const decision = action.evidence.executionReconciliation as ExecutionReconciliation | undefined; @@ -478,8 +510,10 @@ export async function deliverReconciledExecutions( continue; } } - if (!await agentHasRoom(action, action.returnOwnerAgentId)) continue; - admittedByCompany.set(action.companyId, (admittedByCompany.get(action.companyId) ?? 0) + 1); + if (!await agentHasRoom(action, action.returnOwnerAgentId)) { + seenAgentIds.add(action.returnOwnerAgentId); + continue; + } const run = await wake(action.returnOwnerAgentId, { source: "automation", triggerDetail: "system", @@ -501,35 +535,42 @@ export async function deliverReconciledExecutions( ? { freshContinuationIsolation: action.id } : {}), }, }); - if (run) - await db.transaction(async (tx) => { - await tx - .update(heartbeatRuns) - .set({ retryOfRunId: decision.runId }) - .where( - and( - eq(heartbeatRuns.companyId, action.companyId), - eq(heartbeatRuns.id, run.id), - eq(heartbeatRuns.agentId, action.returnOwnerAgentId!), - sql`${heartbeatRuns.contextSnapshot}->>'recoveryActionId' = ${action.id}`, - sql`${heartbeatRuns.contextSnapshot}->>'previousRunId' = ${decision.runId}`, - ), - ); - await tx - .update(issueRecoveryActions) - .set({ - evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify( - { - continuationDelivery: "delivered", - continuationRunId: run.id, - }, - )}::jsonb`, - }) - .where(pendingDecision); - await settleCoalescedFreshContinuationHolds(tx as unknown as Db, { companyId: action.companyId, primaryId: action.id, - delivery: "delivered", continuationRunId: run.id }); - }); + seenAgentIds.add(action.returnOwnerAgentId); + if (!run) { + await deferRefusedContinuation(action); + continue; + } + admittedByCompany.set(action.companyId, (admittedByCompany.get(action.companyId) ?? 0) + 1); + await db.transaction(async (tx) => { + await tx + .update(heartbeatRuns) + .set({ retryOfRunId: decision.runId }) + .where( + and( + eq(heartbeatRuns.companyId, action.companyId), + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.agentId, action.returnOwnerAgentId!), + sql`${heartbeatRuns.contextSnapshot}->>'recoveryActionId' = ${action.id}`, + sql`${heartbeatRuns.contextSnapshot}->>'previousRunId' = ${decision.runId}`, + ), + ); + await tx + .update(issueRecoveryActions) + .set({ + evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify( + { + continuationDelivery: "delivered", + continuationRunId: run.id, + }, + )}::jsonb`, + }) + .where(pendingDecision); + await settleCoalescedFreshContinuationHolds(tx as unknown as Db, { companyId: action.companyId, primaryId: action.id, + delivery: "delivered", continuationRunId: run.id }); + }); } catch { + if (action.returnOwnerAgentId) seenAgentIds.add(action.returnOwnerAgentId); + await deferRefusedContinuation(action).catch(() => undefined); logger.warn( { recoveryActionId: action.id }, "Reconciled execution continuation remains pending for retry", diff --git a/server/src/services/recovery-continuation-pacing.ts b/server/src/services/recovery-continuation-pacing.ts new file mode 100644 index 00000000000..266e795b15f --- /dev/null +++ b/server/src/services/recovery-continuation-pacing.ts @@ -0,0 +1,12 @@ +export const RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS = 5; +export const RECOVERY_CONTINUATION_MAX_HOLD_MS = 5 * 60_000; +export const RECOVERY_CONTINUATION_DELIVERY_BOUND_MS = 15 * 60_000; + +export function recoveryContinuationHeldSince(input: { + evidence: Record; + resolvedAt: Date | null; + updatedAt: Date; +}) { + const recordedAt = Date.parse(String((input.evidence.freshContinuation as { recordedAt?: unknown } | undefined)?.recordedAt ?? "")); + return Number.isFinite(recordedAt) ? recordedAt : (input.resolvedAt ?? input.updatedAt).getTime(); +} diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index 8ee3505183c..01a72a947c4 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -11,6 +11,7 @@ import { EXECUTION_RECONCILIATION_CAUSES, HEARTBEAT_RUN_STATUSES, readProviderFa import { captureTaskInterruptionFailure } from "../sentry.js"; import { isAiAuthenticationBlocked } from "./ai-auth-failure.js"; import { readSystemRecovery } from "./system-recovery-state.js"; +import { RECOVERY_CONTINUATION_DELIVERY_BOUND_MS, recoveryContinuationHeldSince } from "./recovery-continuation-pacing.js"; import { PROJECT_ACCESS_DENIED } from "./project-access-denial.js"; import { authorizationService } from "./authorization.js"; import { taskWorkspaceProjectRequirements } from "./task-workspace-access.js"; @@ -582,6 +583,8 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i runId: sql`${issueRecoveryActions.evidence}->>'runId'`, delivery: sql`${issueRecoveryActions.evidence}->>'continuationDelivery'`, recorded: sql`${issueRecoveryActions.evidence} ? 'freshContinuation'`, + evidence: sql>`jsonb_build_object('freshContinuation', ${issueRecoveryActions.evidence}->'freshContinuation')`, + resolvedAt: issueRecoveryActions.resolvedAt, updatedAt: issueRecoveryActions.updatedAt, checkKind: sql`${issueRecoveryActions.evidence}#>>'{freshContinuationCheck,kind}'`, nextCheckAt: sql`${issueRecoveryActions.evidence}#>>'{freshContinuationCheck,nextCheckAt}'` }) .from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.sourceIssueId, ids), @@ -591,7 +594,8 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i const rows = freshRows.filter(row => row.issueId === issueId && row.runId === runId); // Only the primary record (it carries `freshContinuation`) says whether a // continuation is still on its way; holds coalesced into it never do. - if (rows.some(row => row.recorded && ["pending", "not_requested"].includes(row.delivery ?? ""))) return { state: "continuing" }; + if (rows.some(row => row.recorded && (row.delivery === "not_requested" || (row.delivery === "pending" + && Date.now() - recoveryContinuationHeldSince(row) < RECOVERY_CONTINUATION_DELIVERY_BOUND_MS)))) return { state: "continuing" }; const scheduled = rows.filter(row => ["exact_recovery_pending", "backoff"].includes(row.checkKind ?? "") && row.nextCheckAt) .map(row => Number(row.nextCheckAt)).sort((a, b) => a - b)[0]; return scheduled ? { state: "scheduled", nextCheckAt: scheduled } : undefined;