diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 684f41782ed..65578e0ac47 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -64,6 +64,7 @@ import { prepareHeartbeatWorkspace, RetryWorkspaceRefused } from "./heartbeat/wo import { nativeRecoveryClaimRelinquished, ownNativeExecutionLeaseLapsed, releaseLapsedNativeRestartClaim } from "./native-runtime/native-execution-lease.js"; import { executeHeartbeatRuntime, NativeSessionResumeScheduledError, NativeWorkspaceFinalizeScheduledError } from "./heartbeat/runtime-execution.js"; import { preserveNativeFailedWorkspace } from "./native-runtime/native-failed-workspace-recovery.js"; +import { startOverdueWorkspaceRecovery } from "./workspace-recovery.js"; import { nativeRunHasNoWorkspaceAccess } from "./workspace-restore-recovery-state.js"; import { prepareAndSelectHeartbeatRuntime } from "./heartbeat/runtime-selection.js"; import { createHeartbeatRunCompletion } from "./heartbeat/run-completion.js"; @@ -1028,6 +1029,7 @@ export function heartbeatService( liveRunExecutions, // Bound at call time: the heartbeat recovery module is created below. settleLostRunProcess: scope => recoverActiveExecutions(scope), + startSavedWorkRecovery: source => startOverdueWorkspaceRecovery(db, environmentRuntime, resumeRemoteStopComments, source), settleExplicitContinuationRetry: releaseIssueExecutionAndPromote, scheduleRecoveryRetry: async (runId, outcome) => { const [run] = await db diff --git a/server/src/services/recovery/assigned-work-invariant.test.ts b/server/src/services/recovery/assigned-work-invariant.test.ts index 6f56b1e550b..5bebab6a487 100644 --- a/server/src/services/recovery/assigned-work-invariant.test.ts +++ b/server/src/services/recovery/assigned-work-invariant.test.ts @@ -9,13 +9,16 @@ import { SETTLING_MAX_MS, STALE_BLOCK_MS, SYSTEM_RECOVERY_WORKING_BOUND_MS, + SAVED_WORK_RECOVERY_START_BOUND_MS, VIOLATION_REASONS, type AssignedWorkFacts, + type SavedWorkRecoveryFact, } from "./assigned-work-invariant.js"; const now = new Date("2026-11-12T14:35:00Z"); const ago = (ms: number) => new Date(now.getTime() - ms); const AGENT = "agent-1"; +const HOUR = 60 * 60_000; type Overrides = Omit, "issue"> & { issue?: Partial }; function facts(overrides: Overrides = {}): AssignedWorkFacts { @@ -114,6 +117,49 @@ describe("assigned-work invariant", () => { }); }); + describe("a saved-work recovery the recovery queue owes", () => { + // Staging, 2026-11-13: a legacy retry failed in setup, its files are held + // for recovery, the queue never attempted it, and nothing else is pending. + const stagingHold = (stoppedAgoMs: number, recovery: Partial = {}): Overrides => ({ + latestRun: { id: "run-1", status: "failed", errorCode: "setup_failed", finishedAt: ago(stoppedAgoMs), + settlingUntil: null, operatorStopped: false }, + interruption: { responsibility: "system", state: "failed", userAction: null, nextRetryAt: null, canRetry: false }, + savedWorkRecovery: { actionId: "hold-1", runId: "run-1", state: "scheduled", nextAttemptAt: null, activeAt: null, + sourceFinishedAt: ago(stoppedAgoMs), ...recovery }, + }); + + it("is the task's pending path while the queue has time to start it", () => { + expect(evaluateAssignedWork(facts(stagingHold(SAVED_WORK_RECOVERY_START_BOUND_MS - 60_000)), now)) + .toEqual({ kind: "ok", clause: "pending_intent", intent: "saved_work_recovery" }); + }); + + it("is overdue, not 'nothing pending', once the queue has not started it in time", () => { + expect(evaluateAssignedWork(facts(stagingHold(45 * 60_000)), now)) + .toEqual({ kind: "violation", reason: "saved_work_recovery_overdue", stale: false }); + // Blocked by the same hold: the recovery is still what the task waits on. + expect(evaluateAssignedWork(facts({ ...stagingHold(45 * 60_000), issue: { status: "blocked" }, blockedBy: "system" }), now)) + .toMatchObject({ kind: "violation", reason: "saved_work_recovery_overdue" }); + }); + + it("stays pending while an attempt runs or its next attempt is scheduled", () => { + expect(evaluateAssignedWork(facts(stagingHold(3 * HOUR, { state: "running", activeAt: now.getTime() - 10_000 })), now)) + .toMatchObject({ kind: "ok", intent: "saved_work_recovery" }); + expect(evaluateAssignedWork(facts(stagingHold(3 * HOUR, { nextAttemptAt: now.getTime() + 60_000 })), now)) + .toMatchObject({ kind: "ok", intent: "saved_work_recovery" }); + // A retry that became due recently gets the whole bound again. + expect(evaluateAssignedWork(facts(stagingHold(3 * HOUR, { nextAttemptAt: now.getTime() - 60_000 })), now)) + .toMatchObject({ kind: "ok", intent: "saved_work_recovery" }); + // An attempt that stopped renewing is not running. + expect(evaluateAssignedWork(facts(stagingHold(3 * HOUR, { state: "running", activeAt: now.getTime() - SAVED_WORK_RECOVERY_START_BOUND_MS - 1 })), now)) + .toMatchObject({ kind: "violation", reason: "saved_work_recovery_overdue" }); + }); + + it("leaves someone else's move as the wait", () => { + expect(evaluateAssignedWork(facts({ ...stagingHold(45 * 60_000), pendingInteractions: 1 }), now)) + .toEqual({ kind: "ok", clause: "waiting", reason: "question" }); + }); + }); + describe("clause 3: someone else's move", () => { const cases: Array<[string, Overrides]> = [ ["question", { pendingInteractions: 1 }], diff --git a/server/src/services/recovery/assigned-work-invariant.ts b/server/src/services/recovery/assigned-work-invariant.ts index 4c459622433..c94361b777f 100644 --- a/server/src/services/recovery/assigned-work-invariant.ts +++ b/server/src/services/recovery/assigned-work-invariant.ts @@ -26,6 +26,7 @@ export const PENDING_INTENTS = [ "pending_handoff", "session_goal", "system_recovery_working", + "saved_work_recovery", ] as const; export type PendingIntent = (typeof PENDING_INTENTS)[number]; @@ -64,6 +65,10 @@ export const VIOLATION_REASONS = [ /** Blocked, and nothing recorded says why. */ "unknown_block", "blocked_with_live_run", + /** The task's saved files must be recovered before it can run, and the + * recovery queue has not started that recovery within its bound. Repaired + * by starting the recovery, never by a wake its hold would refuse. */ + "saved_work_recovery_overdue", ] as const; export type ViolationReason = (typeof VIOLATION_REASONS)[number]; @@ -102,6 +107,12 @@ export const SETTLING_GRACE_MS = 90_000; /** Upper bound on how long another server's lease can keep a finished run settling. */ export const SETTLING_MAX_MS = 5 * 60_000; export const SYSTEM_RECOVERY_WORKING_BOUND_MS = 17 * 60_000; +/** How long a queued saved-work recovery may wait for the recovery queue to + * start it, counted from the run stopping or its last attempt, before the + * sweep starts it itself. */ +export const SAVED_WORK_RECOVERY_START_BOUND_MS = 10 * 60_000; +/** A running saved-work recovery renews its observation every 15 seconds. */ +const SAVED_WORK_RECOVERY_ACTIVE_MS = 45_000; /** Blocked tasks older than this are reported, never continued automatically. */ export const STALE_BLOCK_MS = 7 * 24 * 60 * 60_000; /** Automatic continuations per task, shared with the server-restart lane. */ @@ -161,6 +172,9 @@ export type AssignedWorkFacts = { reviewPending: boolean; activeRecoveryAction: { cause: string; ownerType: string; resumeAt: Date | null; continuedBySweep?: boolean; id?: string } | null; + /** A saved-work recovery the recovery queue still owes this task: the + * task's saved files must be recovered before any run can start. */ + savedWorkRecovery?: SavedWorkRecoveryFact | null; agent: { status: string; invokable: boolean } | null; companyPaused: boolean; pauseHold: boolean; @@ -169,6 +183,33 @@ export type AssignedWorkFacts = { blockedBy: "user" | "assignee" | "system" | "unknown" | null; }; +export type SavedWorkRecoveryFact = { + actionId: string; + runId: string; + state: "scheduled" | "running" | "awaiting_evidence"; + /** When the next attempt is due (epoch ms); null when due now. */ + nextAttemptAt: number | null; + /** Last observation of a running attempt (epoch ms). */ + activeAt: number | null; + /** When the recovered run stopped. */ + sourceFinishedAt: Date | null; +}; + +/** Whether the recovery queue still has time to start or finish this + * recovery: an attempt is running, the next one is scheduled, or it became + * due less than SAVED_WORK_RECOVERY_START_BOUND_MS ago. */ +export function savedWorkRecoveryPending(recovery: SavedWorkRecoveryFact, now: Date): boolean { + const at = now.getTime(); + if (recovery.activeAt !== null && recovery.activeAt > at - SAVED_WORK_RECOVERY_ACTIVE_MS) return true; + if (recovery.nextAttemptAt !== null && recovery.nextAttemptAt > at) return true; + return at - savedWorkRecoveryDueSince(recovery) < SAVED_WORK_RECOVERY_START_BOUND_MS; +} + +/** When the queue became responsible for the next attempt. */ +export function savedWorkRecoveryDueSince(recovery: SavedWorkRecoveryFact): number { + return Math.max(recovery.sourceFinishedAt?.getTime() ?? 0, recovery.nextAttemptAt ?? 0, recovery.activeAt ?? 0); +} + export type AssignedWorkVerdict = | { kind: "not_applicable" } | { kind: "ok"; clause: "live_run"; runId: string } @@ -216,6 +257,10 @@ export function evaluateAssignedWork(facts: AssignedWorkFacts, now: Date = new D const waiting = waitingReason(facts, now); if (waiting) return { kind: "ok", clause: "waiting", reason: waiting }; + // The saved files must be recovered first, and the queue has not reached + // them in time. A continuation wake would only wait behind the same hold. + if (facts.savedWorkRecovery) return { kind: "violation", reason: "saved_work_recovery_overdue", stale: false }; + if (issue.status === "blocked") { const blockedAt = issue.blockedTransitionAt ?? issue.updatedAt; const stale = now.getTime() - blockedAt.getTime() > STALE_BLOCK_MS; @@ -237,6 +282,7 @@ function pendingIntent(facts: AssignedWorkFacts, now: Date): PendingIntent | nul if (facts.pendingHandoff) return "pending_handoff"; if (facts.sessionGoal === "active") return "session_goal"; if (facts.issue.monitorNextCheckAt && facts.issue.monitorNextCheckAt.getTime() > now.getTime()) return "scheduled_monitor"; + if (facts.savedWorkRecovery && savedWorkRecoveryPending(facts.savedWorkRecovery, now)) return "saved_work_recovery"; const interruption = facts.interruption; if (interruption?.responsibility === "system") { if (interruption.state === "working" && !(facts.latestRun?.finishedAt @@ -345,6 +391,8 @@ export function continuationActivityMessage(reason: ViolationReason): string { return "Continued automatically after an interruption. Paperclip queued the task again, so you don't need to press Retry."; case "blocked_with_live_run": return "No longer blocked: this task is already queued to continue, so nothing needs you."; + case "saved_work_recovery_overdue": + return "Paperclip started recovering this task's saved files. The task continues once they are recovered."; default: return "Continued automatically after an interruption. Nothing was running or scheduled, so Paperclip queued the task again."; } diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index c362330a984..03f8843f32b 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -29,6 +29,7 @@ import { issueTreeControlService } from "../issue-tree-control.js"; import { taskInterruptionPresentations } from "../task-interruption-presentation.js"; import { CHAT_CONTROL_RECOVERY_STOP_CODE, CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE } from "../chat-control-recovery-stop.js"; import { onboardingFirstTaskAwaitingUserSql } from "./onboarding-first-task-wait.js"; +import { workspaceRecoverySourceEligible } from "../workspace-recovery-source.js"; import { PENDING_CONTINUATION_BOUND_MS } from "../recovery-continuation-pacing.js"; import { attributeBlock, BLOCK_ATTRIBUTION_WINDOW_MS, blockedStatusDetailsSql } from "../interruption-block-release.js"; import { ASSIGNED_WORK_CONTINUED_ACTION, taskAutomaticAttempts } from "./task-automatic-attempts.js"; @@ -37,6 +38,8 @@ import { continuationActivityMessage, continuationAllowance, CONTINUATION_PENDING_MESSAGE, + SAVED_WORK_RECOVERY_START_BOUND_MS, + savedWorkRecoveryDueSince, CONTINUATION_REFUSED_MESSAGE, CONTINUATION_REVIEW_MESSAGE, evaluateAssignedWork, @@ -45,6 +48,7 @@ import { type AssignedWorkFacts, type AssignedWorkVerdict, type RunOwnerVerdict, + type SavedWorkRecoveryFact, type ViolationReason, } from "./assigned-work-invariant.js"; @@ -84,6 +88,9 @@ export type AssignedWorkDeps = { liveRunExecutions?: Readonly<{ has(id: string): boolean }>; /** Heartbeat's lost-process settle path for a running run whose owner is gone. */ settleLostRunProcess?: (scope: { companyId: string; runId: string; onDeferredDeadline?: (at: number) => void }) => Promise; + /** Starts a saved-work recovery the recovery queue has not reached in time. */ + startSavedWorkRecovery?: (source: { companyId: string; issueId: string; runId: string; actionId: string }) => + Promise<"started" | "in_flight" | "busy" | "not_eligible">; /** Shared owner proof, injectable for tests. */ proveOwner?: (stamp: LocalOwnerStamp) => Promise; now?: () => Date; @@ -91,7 +98,8 @@ export type AssignedWorkDeps = { export type AssignedWorkCheck = { verdict: AssignedWorkVerdict; - outcome: "ok" | "settling" | "repaired" | "settled_lost_run" | "deferred_rate_limit" | "deferred_cap" | "stale_reported" | "left_to_hold_owner" | "left_to_native_recovery" | "changed" | "enqueue_refused"; + outcome: "ok" | "settling" | "repaired" | "settled_lost_run" | "deferred_rate_limit" | "deferred_cap" | "stale_reported" | "left_to_hold_owner" | "left_to_native_recovery" | "changed" | "enqueue_refused" + | "started_saved_work_recovery" | "saved_work_recovery_busy"; retryAt: Date | null; }; @@ -122,7 +130,7 @@ function createPassLog() { } else if (verdict.kind === "violation") { pass.violations[verdict.reason] = (pass.violations[verdict.reason] ?? 0) + 1; } - if (check.outcome === "repaired" || check.outcome === "settled_lost_run") pass.repaired += 1; + if (check.outcome === "repaired" || check.outcome === "settled_lost_run" || check.outcome === "started_saved_work_recovery") pass.repaired += 1; if (check.outcome === "deferred_rate_limit" || check.outcome === "deferred_cap") pass.deferred += 1; if (check.outcome === "stale_reported") pass.stale += 1; if (!timer) { timer = setTimeout(flush, 5_000); timer.unref?.(); } @@ -181,7 +189,8 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { sql`${issueThreadInteractions.issueId}::text`, issueThreadInteractions.createdAt); const [activeRuns, latestRuns, wakes, chatWakes, nativePending, handoffs, goals, interactions, approvalRows, - blockerRows, resolvedBlockerRows, childRows, actionMap, agentRows, companyRows, onboardingRows, blockActivity, interruptions] = await Promise.all([ + blockerRows, resolvedBlockerRows, childRows, actionMap, agentRows, companyRows, onboardingRows, blockActivity, interruptions, + savedWorkRows] = await Promise.all([ database.select(runColumns).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.status, NON_TERMINAL_RUN_STATUSES), sql`${runIssue} = any(${idParam})`)), // The newest run per task, read through the (company, issueId, created) index. @@ -274,6 +283,20 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { ) latest`) : [], taskInterruptionPresentations(database, companyId, issueRows.map(issue => ({ id: issue.id, companyId: issue.companyId, status: issue.status, assigneeAgentId: issue.assigneeAgentId, projectId: issue.projectId }))), + // Saved-work recoveries the recovery queue still owes: exactly what its + // worker selects, so "pending" means the worker will start it. + database.select({ issueId: issueRecoveryActions.sourceIssueId, actionId: issueRecoveryActions.id, runId: heartbeatRuns.id, + finishedAt: heartbeatRuns.finishedAt, + state: sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,state}'`, + nextAttemptAt: sql`${issueRecoveryActions.evidence}#>'{systemRecovery,nextAttemptAt}'`, + activeAt: sql`${issueRecoveryActions.evidence}->'systemRecoveryActiveAt'` }) + .from(issueRecoveryActions).innerJoin(heartbeatRuns, and(eq(heartbeatRuns.companyId, issueRecoveryActions.companyId), + sql`${heartbeatRuns.id}::text = ${issueRecoveryActions.evidence}->>'runId'`)) + .where(and(eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.sourceIssueId, ids), + inArray(heartbeatRuns.status, ["failed", "interrupted", "timed_out", "cancelled", "succeeded"]), + workspaceRecoverySourceEligible(), + sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,state}' in ('scheduled', 'running', 'awaiting_evidence')`)) + .orderBy(desc(issueRecoveryActions.createdAt)), ]); const rowsOf = (value: unknown): T[] => Array.isArray(value) ? value as T[] : ((value as { rows?: T[] })?.rows ?? []); @@ -321,6 +344,14 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { && row.updatedAt.getTime() < at.getTime() - NATIVE_STALL_MS).map(row => row.runId)); const nativeIds = new Set(nativePending.filter(row => !stalledRunIds.has(row.runId)).map(row => row.issueId)); const handoffIds = new Set(handoffs.map(row => row.issueId)); + const epochMs = (value: unknown) => typeof value === "number" && Number.isFinite(value) ? value : null; + const savedWorkByIssue = new Map(); + for (const row of savedWorkRows) { + if (savedWorkByIssue.has(row.issueId)) continue; + savedWorkByIssue.set(row.issueId, { actionId: row.actionId, runId: row.runId, + state: row.state as SavedWorkRecoveryFact["state"], nextAttemptAt: epochMs(row.nextAttemptAt), activeAt: epochMs(row.activeAt), + sourceFinishedAt: row.finishedAt ? new Date(row.finishedAt) : null }); + } const interactionCounts = counts(interactions); const approvalCounts = counts(approvalRows); const blockerCounts = counts(blockerRows); @@ -397,6 +428,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { pauseHold, budgetStop: companyBudget || agent?.pauseReason === "budget" || agent?.pauseReason === "budget_exceeded", blockedBy: issue.status !== "blocked" ? null : attributeBlock(blockedEntry, issue.assigneeAgentId), + savedWorkRecovery: savedWorkByIssue.get(issue.id) ?? null, }); } return result; @@ -476,6 +508,10 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { sql`not exists (select 1 from issue_recovery_actions ra where ra.company_id = ${issues.companyId} and ra.source_issue_id = ${issues.id} and ra.status in ('active', 'escalated') and (ra.cause in ('configuration_incomplete', 'ownership_owner_unprovable') or ${issues.status} = 'blocked') + -- except a saved-work recovery the queue still owes: the predicate + -- counts it pending for its bound and starts it once it is overdue + and not (${issues.status} = 'blocked' and ra.cause not in ('configuration_incomplete', 'ownership_owner_unprovable') + and coalesce(ra.evidence#>>'{systemRecovery,state}', '') in ('scheduled', 'running', 'awaiting_evidence')) -- except a recent board escalation, which the predicate judges by its failed run's cause and not (ra.kind = ${STRANDED_KIND} and ra.owner_type = 'board' and exists (select 1 from heartbeat_runs sr where sr.company_id = ra.company_id and sr.id::text = ra.evidence->>'latestRunId' @@ -505,12 +541,21 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { } async function act(issue: IssueRow, facts: AssignedWorkFacts, verdict: AssignedWorkVerdict, at: Date): Promise { + // A queued saved-work recovery is pending only for its bound: look again + // when the bound ends, so an overdue recovery is started even if nothing + // else wakes this task. + if (verdict.kind === "ok" && verdict.clause === "pending_intent" && verdict.intent === "saved_work_recovery" && facts.savedWorkRecovery) + return { verdict, outcome: "ok", retryAt: new Date(savedWorkRecoveryDueSince(facts.savedWorkRecovery) + SAVED_WORK_RECOVERY_START_BOUND_MS + 1_000) }; if (verdict.kind === "not_applicable" || verdict.kind === "ok") return { verdict, outcome: "ok", retryAt: null }; if (verdict.kind === "settling") return { verdict, outcome: "settling", retryAt: verdict.recheckAt }; if (verdict.stale) return { verdict, outcome: "stale_reported", retryAt: null }; if (verdict.reason === "blocked_with_live_run") { return { verdict, outcome: await liftBlockBehindLiveRun(issue, at) ? "repaired" : "changed", retryAt: null }; } + // The saved files come first: start their overdue recovery. Its hold + // refuses any wake until it completes, and its completion continues the + // task (or, if the files cannot be recovered, offers a working Retry). + if (verdict.reason === "saved_work_recovery_overdue") return startSavedWorkRecovery(issue, facts, verdict, at); // A task blocked under an active recovery hold is re-continued by the // hold's owner (the fresh-continuation fallback, same per-task cap). The // sweep reports it and repairs only blocks that have no hold at all, so @@ -563,6 +608,26 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { return { verdict: claimed.verdict, outcome: run ? "repaired" : "enqueue_refused", retryAt: claimed.nextCheckAt }; } + async function startSavedWorkRecovery(issue: IssueRow, facts: AssignedWorkFacts, + verdict: Extract, at: Date): Promise { + const recovery = facts.savedWorkRecovery!; + const recheckAt = new Date(at.getTime() + SAVED_WORK_RECOVERY_START_BOUND_MS); + if (!deps.startSavedWorkRecovery) return { verdict, outcome: "saved_work_recovery_busy", retryAt: recheckAt }; + const started = await deps.startSavedWorkRecovery({ companyId: issue.companyId, issueId: issue.id, + runId: recovery.runId, actionId: recovery.actionId }).catch(error => { + logger.warn({ err: error, companyId: issue.companyId, issueId: issue.id, runId: recovery.runId }, + "assigned-work sweep could not start the overdue saved-work recovery"); + return "busy" as const; + }); + logger.info({ event: "assigned_work_invariant.saved_work_recovery", companyId: issue.companyId, issueId: issue.id, + runId: recovery.runId, recoveryActionId: recovery.actionId, result: started, + waitedMs: at.getTime() - savedWorkRecoveryDueSince(recovery) }, "assigned-work sweep found an overdue saved-work recovery"); + // A busy limit or another server's attempt is looked at again soon; a + // started attempt gets the whole bound to report progress. + const retryAt = started === "started" || started === "in_flight" ? recheckAt : new Date(at.getTime() + 60_000); + return { verdict, outcome: started === "started" ? "started_saved_work_recovery" : "saved_work_recovery_busy", retryAt }; + } + type Claimed = { kind: "claimed"; nextCheckAt: Date; activityId: string; publications: ActivityPublication[]; verdict: Extract; reason: ViolationReason; restoredTo: "todo" | "in_progress" | "in_review" | null; restoredStatusVersion: number | null; diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 568e46ff21c..d2dbce5c7b1 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -33,7 +33,7 @@ import { externalConversationStateSql } from "../slack-conversation-state.js"; import { isIssueReviewPathRecoveryRun } from "./review-path-recovery.js"; import { escalateExhaustedIssueReviewPathRecovery } from "./review-path-recovery-escalation.js"; import { executionRetryAccounting, INTERRUPTED_PREPARATION_RETRY_REASON } from "../execution-recovery-attempt.js"; -import { assignedWorkSweep } from "./assigned-work-sweep.js"; +import { assignedWorkSweep, type AssignedWorkDeps } from "./assigned-work-sweep.js"; import { isExplicitContinuationRetryClaim } from "../explicit-continuation-retry-claim.js"; import { decideLegacyContinuation, legacyDispositionEpisode, legacyDispositionFingerprint, @@ -996,6 +996,8 @@ export function recoveryService( * judging process death itself. */ settleLostRunProcess?: (scope: { companyId: string; runId: string; onDeferredDeadline?: (at: number) => void }) => Promise; + /** Starts a saved-work recovery the recovery queue has not reached in time. */ + startSavedWorkRecovery?: AssignedWorkDeps["startSavedWorkRecovery"]; beforeOrphanedRunTerminalWrite?: (runId: string) => Promise; beforeStaleLockClear?: (issueId: string) => Promise; afterStrandedLatestRunRead?: (input: { issueId: string; runId: string | null }) => Promise; @@ -6858,6 +6860,7 @@ export function recoveryService( enqueueWakeup: (agentId, opts) => deps.enqueueWakeup(agentId, opts), liveRunExecutions: deps.liveRunExecutions, settleLostRunProcess: deps.settleLostRunProcess, + startSavedWorkRecovery: deps.startSavedWorkRecovery, }); return { diff --git a/server/src/services/saved-work-recovery-overdue.test.ts b/server/src/services/saved-work-recovery-overdue.test.ts new file mode 100644 index 00000000000..eb583d68320 --- /dev/null +++ b/server/src/services/saved-work-recovery-overdue.test.ts @@ -0,0 +1,159 @@ +import { and, eq, sql } from "drizzle-orm"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { activityLog, heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; +import { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { registerIssueRecoveryWork } from "./recovery/work.js"; +import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; +import { readSystemRecovery } from "./system-recovery-state.js"; +import { startOverdueWorkspaceRecovery } from "./workspace-recovery.js"; +import { getExecutionBlocker } from "./execution-blocker.js"; +import { heartbeatService } from "./heartbeat.js"; +import { deliverReconciledExecutions } from "./execution-recovery-resolution.js"; +import { assignedWorkSweep, ASSIGNED_WORK_CONTINUED_ACTION } from "./recovery/assigned-work-sweep.js"; +import { SAVED_WORK_RECOVERY_START_BOUND_MS } from "./recovery/assigned-work-invariant.js"; +import { action, connectionString, db, failedRemotePreparation, useWorkspaceRecoveryFixture } from "./__tests__/workspace-recovery-fixture.js"; + +const targets = vi.hoisted(() => new Map()); +vi.mock("./environment-execution-target.js", async importOriginal => { + const actual = await importOriginal(); + return { ...actual, resolveEnvironmentExecutionTarget: async (input: Parameters[0]) => + (input.leaseId ? targets.get(input.leaseId) : undefined) ?? actual.resolveEnvironmentExecutionTarget(input) }; +}); + +/** + * Staging, 2026-11-13: a legacy retry failed in setup when saving + * its sandbox workspace timed out (`setup_failed`, "capture (configured + * timeout 300000ms)"). Its files were held for a saved-work recovery + * (`legacy_execution_requires_reconciliation`, resolved, replay blocked) that + * the recovery queue still had at 0 attempts 45 minutes later; 29 such + * recoveries waited on the instance, one since 15:01Z. The task said + * "Recovering saved work…" while the assigned-work sweep logged + * `no_pending_path` and a continuation the hold refused, every few minutes. + */ +describe("a saved-work recovery the queue has not reached", () => { + useWorkspaceRecoveryFixture(targets); + beforeEach(() => { + vi.stubEnv("DATABASE_URL", connectionString); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "63".repeat(32)); + }); + afterEach(() => { vi.unstubAllEnvs(); }); + + async function stagingShape(stoppedAgoMs: number, clock: { now?: () => Date } = {}) { + const f = await failedRemotePreparation(); + const finishedAt = new Date(Date.now() - stoppedAgoMs); + await db.update(heartbeatRuns).set({ finishedAt, startedAt: new Date(finishedAt.getTime() - 10 * 60_000) }) + .where(eq(heartbeatRuns.id, f.runId)); + await db.update(issues).set({ status: "in_progress", statusVersion: 3 }).where(eq(issues.id, f.issueId)); + // The hold the staging task had: adopted by the system, never attempted. + expect(readSystemRecovery((await action(f.actionId)).evidence)).toMatchObject({ state: "scheduled", attempts: 0 }); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).not.toBeNull(); + const onRecovered = vi.fn(async () => undefined); + // Staging's admission parked every continuation behind the hold. + const enqueueWakeup = vi.fn(async () => null); + const sweep = assignedWorkSweep(db, { enqueueWakeup, ...clock, + startSavedWorkRecovery: source => startOverdueWorkspaceRecovery(db, f.environmentRuntime, onRecovered, source) }); + return { ...f, sweep, enqueueWakeup, onRecovered }; + } + + const continuationEntries = async (issueId: string) => db.select().from(activityLog).where(and( + eq(activityLog.entityId, issueId), eq(activityLog.action, ASSIGNED_WORK_CONTINUED_ACTION))); + + it("starts the recovery itself once it is overdue, and the task's files are recovered", async () => { + const f = await stagingShape(45 * 60_000); + const check = await f.sweep.checkIssue({ companyId: f.companyId, issueId: f.issueId }); + expect(check).toMatchObject({ verdict: { kind: "violation", reason: "saved_work_recovery_overdue" }, + outcome: "started_saved_work_recovery" }); + // No wake that the hold would refuse, and no "could not start" entry. + expect(f.enqueueWakeup).not.toHaveBeenCalled(); + expect(await continuationEntries(f.issueId)).toHaveLength(0); + await vi.waitFor(async () => { + expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete"); + }, { timeout: 20_000, interval: 100 }); + // The recovery's completion hands the run to the normal continuation path. + await vi.waitFor(() => expect(f.onRecovered).toHaveBeenCalledOnce(), { timeout: 5_000 }); + const [recovered] = f.onRecovered.mock.calls[0] as unknown as [{ id: string }]; + expect(recovered.id).toBe(f.runId); + expect((await getExecutionBlocker(db, f.companyId, f.issueId))?.workspaceRecovery?.status).toBe("recovered"); + // With the files recovered the task continues: the next check admits an + // ordinary continuation through the real wake path. + const heartbeat = heartbeatService(db); + const continued = assignedWorkSweep(db, { enqueueWakeup: (agentId, opts) => heartbeat.wakeup(agentId, opts) }); + // The hold now lets a continuation through: the sweep records a fresh + // continuation from the recovered files, and the continuation worker + // starts its run. + await continued.checkIssue({ companyId: f.companyId, issueId: f.issueId }); + expect(await continued.checkIssue({ companyId: f.companyId, issueId: f.issueId })) + .toMatchObject({ verdict: { kind: "ok", clause: "pending_intent" } }); + await deliverReconciledExecutions(db, (agentId, opts) => heartbeat.wakeup(agentId, opts)); + const [successor] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${f.issueId}`, sql`${heartbeatRuns.id} <> ${f.runId}`)); + expect(successor).toBeDefined(); + }, 90_000); + + it("counts a recovery the queue has time to start as the task's pending path, and looks again when its bound ends", async () => { + let clock = Date.now(); + const f = await stagingShape(2 * 60_000, { now: () => new Date(clock) }); + const stoppedAt = (await db.select({ finishedAt: heartbeatRuns.finishedAt }).from(heartbeatRuns) + .where(eq(heartbeatRuns.id, f.runId)))[0].finishedAt!.getTime(); + const deadlines: number[] = []; + expect(await f.sweep.checkIssue({ companyId: f.companyId, issueId: f.issueId, onDeferredDeadline: at => deadlines.push(at) })) + .toMatchObject({ verdict: { kind: "ok", clause: "pending_intent", intent: "saved_work_recovery" }, outcome: "ok" }); + expect(f.enqueueWakeup).not.toHaveBeenCalled(); + expect(readSystemRecovery((await action(f.actionId)).evidence)).toMatchObject({ state: "scheduled", attempts: 0 }); + expect(await continuationEntries(f.issueId)).toHaveLength(0); + // The worker keeps this deadline; nothing else needs to wake the task. + expect(deadlines).toEqual([stoppedAt + SAVED_WORK_RECOVERY_START_BOUND_MS + 1_000]); + clock = deadlines[0]; + expect(await f.sweep.checkIssue({ companyId: f.companyId, issueId: f.issueId })) + .toMatchObject({ verdict: { reason: "saved_work_recovery_overdue" }, outcome: "started_saved_work_recovery" }); + await vi.waitFor(async () => { + expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete"); + }, { timeout: 20_000, interval: 100 }); + }, 60_000); + + it("never starts a second attempt beside one already running for the same run", async () => { + const f = await stagingShape(SAVED_WORK_RECOVERY_START_BOUND_MS + 60_000); + const source = { companyId: f.companyId, issueId: f.issueId, runId: f.runId, actionId: f.actionId }; + const first = await startOverdueWorkspaceRecovery(db, f.environmentRuntime, f.onRecovered, source); + const second = await startOverdueWorkspaceRecovery(db, f.environmentRuntime, f.onRecovered, source); + expect([first, second]).toEqual(["started", "in_flight"]); + await vi.waitFor(async () => { + expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete"); + }, { timeout: 20_000, interval: 100 }); + expect(readSystemRecovery((await action(f.actionId)).evidence)?.attempts).toBe(1); + // A completed recovery is not selected again. + await vi.waitFor(async () => { + expect(await startOverdueWorkspaceRecovery(db, f.environmentRuntime, f.onRecovered, source)).toBe("not_eligible"); + }, { timeout: 10_000, interval: 100 }); + const [row] = await db.select({ count: sql`count(*)::int` }).from(heartbeatRuns) + .where(eq(heartbeatRuns.id, f.runId)); + expect(row.count).toBe(1); + }, 60_000); + + it("reaches a blocked task with an active saved-work hold through the recovery worker", async () => { + const f = await stagingShape(45 * 60_000); + // The same hold, still active, on a task the recovery left blocked. + await db.update(issueRecoveryActions).set({ status: "active", outcome: null, resolvedAt: null }) + .where(eq(issueRecoveryActions.id, f.actionId)); + await db.update(issues).set({ status: "blocked", blockedTransitionAt: new Date(Date.now() - 40 * 60_000) }) + .where(eq(issues.id, f.issueId)); + expect((await f.sweep.listSuspects({ companyId: f.companyId }, null, 50)).map(row => row.id)).toContain(f.issueId); + const onError = vi.fn(); + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, canReconcile: () => true, onError }); + const noop = async () => ({ enqueueFailed: 0, incomplete: 0 }); + const worker = registerIssueRecoveryWork(db, coordinator, { stranded: noop, orphanBlockers: noop, recoveryActions: noop, + dependencies: noop, locks: noop, watchdogs: noop, schedulingSuppressed: async () => false, + assignedWork: { listSuspects: f.sweep.listSuspects, checkIssue: f.sweep.checkIssue } }, () => true); + try { + await worker.ready; + await vi.waitFor(async () => { + expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete"); + }, { timeout: 30_000, interval: 250 }); + expect(f.enqueueWakeup).not.toHaveBeenCalled(); + expect(onError).not.toHaveBeenCalled(); + } finally { + await worker.stop(); + await coordinator.stop(); + } + }, 90_000); +}); diff --git a/server/src/services/workspace-recovery.ts b/server/src/services/workspace-recovery.ts index 57fcb89aa20..7d652a35e1f 100644 --- a/server/src/services/workspace-recovery.ts +++ b/server/src/services/workspace-recovery.ts @@ -572,6 +572,10 @@ async function recordCopybackOwnerHold(db: Db, input: Source, generationId: stri const cursors = new WeakMap(); const workInFlight = new WeakMap>(); +/** Overdue recoveries the assigned-work sweep started outside the queue's + * slots. The queue skips their runs; their count has its own small limit. */ +const overdueInFlight = new WeakMap>(); +export const OVERDUE_WORKSPACE_RECOVERY_LIMIT = 2; const unfinishedCursors = new WeakMap(); /** Destruction is a later cleanup obligation, authorized only by a matching @@ -697,7 +701,7 @@ export async function reconcileRetainedWorkspaces(db: Db, runtime: EnvironmentRu if (pending.size >= 2) return; // Locally running exports already own their slot. Exclude them before // pagination so a page containing only held work cannot prevent wrapping. - const inFlightRunIds = [...pending]; + const inFlightRunIds = [...pending, ...(overdueInFlight.get(db) ?? [])]; let unfinishedRows = await retainedWorkspaceCandidates(db, unfinishedCursors.get(db), true, false, inFlightRunIds); if (!unfinishedRows.length) unfinishedRows = await retainedWorkspaceCandidates(db, undefined, true, false, inFlightRunIds); const unfinished: CandidateLane = { rows: unfinishedRows, index: 0, advance: id => unfinishedCursors.set(db, id) }; @@ -764,30 +768,64 @@ export async function reconcileRetainedWorkspaces(db: Db, runtime: EnvironmentRu const finish = beginIdleTrackedWork(); const physical = actions.some(action => ["scheduled", "running", "awaiting_evidence"].includes(readSystemRecovery(action.evidence)?.state ?? "")); work.push((async () => { try { - for (const action of actions) await recoverRunAction(action, run); + for (const action of actions) await recoverRunAction(db, runtime, onRecovered, action, run); } finally { pending.delete(run.id); finish(); onFinished?.(physical || hasUnvisited); } })()); } return work; +} - async function recoverRunAction(action: typeof issueRecoveryActions.$inferSelect, run: typeof heartbeatRuns.$inferSelect) { - try { - await recoverRetainedWorkspace({ db, environmentRuntime: runtime, companyId: action.companyId, issueId: action.sourceIssueId, runId: run.id, actionId: action.id }); - const [updated] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)); - const recovered = readWorkspaceRecovery(updated?.evidence ?? null)?.status === "recovered"; - const exhausted = readSystemRecovery(updated?.evidence)?.state === "exhausted"; - if (recovered || exhausted) { - const [current] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); - if (current) { - if (recovered) { - await resumePreservedResultFinalization(db, current.companyId, current.id); - await scheduleRecoveredWorkspaceCleanup(db, current); - } - // Only normal dispatch can consume existing authorized input. An - // exhausted export permits evaluation of an isolated continuation, - // never another export attempt or a replay of an old assignment. - await onRecovered(current); +/** + * Starts one saved-work recovery that the queue has not reached within its + * bound (the assigned-work sweep's repair). It runs outside the queue's two + * slots, so a queue held by slow or stuck attempts cannot keep a task waiting + * indefinitely, and it goes through the same ownership and attempt bookkeeping + * as the queue, so it never runs beside another attempt for the same run. + * Resolves once the attempt is started (or refused), not when it finishes. + */ +export async function startOverdueWorkspaceRecovery(db: Db, runtime: EnvironmentRuntimeService, + onRecovered: (run: typeof heartbeatRuns.$inferSelect) => Promise, source: Source): + Promise<"started" | "in_flight" | "busy" | "not_eligible"> { + const overdue = overdueInFlight.get(db) ?? new Set(); + overdueInFlight.set(db, overdue); + if (overdue.has(source.runId) || workInFlight.get(db)?.has(source.runId)) return "in_flight"; + if (overdue.size >= OVERDUE_WORKSPACE_RECOVERY_LIMIT) return "busy"; + const [row] = await db.select({ action: issueRecoveryActions, run: heartbeatRuns }).from(issueRecoveryActions) + .innerJoin(heartbeatRuns, and(eq(heartbeatRuns.companyId, issueRecoveryActions.companyId), eq(heartbeatRuns.id, source.runId))) + .where(and(eq(issueRecoveryActions.companyId, source.companyId), eq(issueRecoveryActions.id, source.actionId), + eq(issueRecoveryActions.sourceIssueId, source.issueId), sql`${issueRecoveryActions.evidence}->>'runId' = ${source.runId}`, + inArray(heartbeatRuns.status, terminal), workspaceRecoverySourceEligible(), + sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,state}' in ('scheduled', 'running', 'awaiting_evidence')`)).limit(1); + if (!row) return "not_eligible"; + if (overdue.has(source.runId) || workInFlight.get(db)?.has(source.runId)) return "in_flight"; + overdue.add(source.runId); + const finish = beginIdleTrackedWork(); + void (async () => { + try { await recoverRunAction(db, runtime, onRecovered, row.action, row.run); } + finally { overdue.delete(source.runId); finish(); } + })(); + return "started"; +} + +async function recoverRunAction(db: Db, runtime: EnvironmentRuntimeService, + onRecovered: (run: typeof heartbeatRuns.$inferSelect) => Promise, + action: typeof issueRecoveryActions.$inferSelect, run: typeof heartbeatRuns.$inferSelect) { + try { + await recoverRetainedWorkspace({ db, environmentRuntime: runtime, companyId: action.companyId, issueId: action.sourceIssueId, runId: run.id, actionId: action.id }); + const [updated] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)); + const recovered = readWorkspaceRecovery(updated?.evidence ?? null)?.status === "recovered"; + const exhausted = readSystemRecovery(updated?.evidence)?.state === "exhausted"; + if (recovered || exhausted) { + const [current] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + if (current) { + if (recovered) { + await resumePreservedResultFinalization(db, current.companyId, current.id); + await scheduleRecoveredWorkspaceCleanup(db, current); } + // Only normal dispatch can consume existing authorized input. An + // exhausted export permits evaluation of an isolated continuation, + // never another export attempt or a replay of an old assignment. + await onRecovered(current); } - } catch { logger.warn({ runId: run.id }, "Retained workspace remains pending"); } - } + } + } catch (error) { logger.warn({ err: error, runId: run.id, actionId: action.id }, "Retained workspace remains pending"); } }