import { settleOperatorContainedNativeOwner } from "./native-runtime/native-failed-run-owner-release.js"; import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { notifyChatResponseAvailability } from "./chat-response-availability.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExecutionRunWork } from "./execution-control-notifications.js"; import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; import { randomUUID } from "node:crypto"; import { canonicalExecutionUuid } from "./execution-identity.js"; import { conversationRecoveryActionPredicate, getConversationOwnershipBlocker } from "./conversation-continuation.js"; import { isInterruptedNativePreparationCandidate } from "./cancelled-native-startup.js"; import { persistActivity } from "./activity-log.js"; import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js"; import { foldWaitingRetriesIntoContinuation } from "./fresh-continuation-retry-fold.js"; import { logger } from "../middleware/logger.js"; import { and, asc, eq, exists, inArray, isNull, not, or, sql } from "drizzle-orm"; import { agentWakeupRequests, agents, chatActions, environmentLeases, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, type Db, } from "@paperclipai/db"; import { conflict } from "../errors.js"; import { buildExecutionContinuation } from "./execution-continuation.js"; import { EXECUTION_RECONCILIATION_CAUSES, type ExecutionReconciliation, } from "@paperclipai/shared"; import { parseIssueExecutionState } from "./issue-execution-policy.js"; import { isSupersededConversationRun } from "./agent-conversations.js"; import { hasRequiredWorkspaceRecovery } from "./workspace-restore-recovery-state.js"; import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; import { readSystemRecovery } from "./system-recovery-state.js"; import { provenGoneLegacyControllerBootId } from "./legacy-controller-lease.js"; import { continueFreshInTransaction, continueHeldExecutions, freshContinuationCheckDue, recordFreshContinuationCheck, settleCoalescedFreshContinuationHolds } from "./fresh-continuation-fallback.js"; import { alias } from "drizzle-orm/pg-core"; import { RECOVERY_BACKLOG_AGE_MS, RECOVERY_WAKE_REASONS } from "./heartbeat/queued-run-dispatch-order.js"; import { parseHeartbeatPolicy } from "./heartbeat/timer-policy.js"; import { RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS, RECOVERY_CONTINUATION_BACKLOG_SHARE_PER_PASS, RECOVERY_CONTINUATION_MAX_HOLD_MS, recoveryContinuationHeldSince } from "./recovery-continuation-pacing.js"; export { RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS, RECOVERY_CONTINUATION_DELIVERY_BOUND_MS, RECOVERY_CONTINUATION_MAX_HOLD_MS } from "./recovery-continuation-pacing.js"; const RECOVERY_CONTINUATION_PAGES_PER_PASS = 8; /** An operator records observed outcomes; this is not permission to blindly retry. */ /** * The user-facing note for a settled recovery hold. A restart-recovery hold * names its own cause: a workspace export retried after that hold does not * mean the user must repair the workspace. */ export function executionRecoverySettlementNote(input: { current: boolean; cause: string; workspaceRestoreFailed: boolean; /** The hold's recorded trigger; a same-run retry is not a server restart. */ trigger?: unknown; }): string { if (!input.current) return "Recovery closed because the task's owner, execution, or status changed. No work was replayed."; if (input.cause === "native_restart_recovery_blocked" && input.trigger === "same_run_retry") { return "Automatic retry of this run stopped. Saved work is preserved; actions with unverified outcomes will not be repeated."; } if (input.cause === "native_restart_recovery_blocked") { return "Automatic recovery stopped after a server restart because the previous execution could not be verified. Saved work is preserved; actions with unverified outcomes will not be repeated."; } return input.workspaceRestoreFailed ? "Workspace repair required. Verify safe staging or repair before continuing. Saved work and approval decisions remain in force." : "Automatic recovery stopped. Recorded work is preserved; actions with unverified outcomes will not be repeated."; } type ExecutionReconciliationInput = { db: Db; companyId: string; issueId: string; agentId: string | null; sourceRunId: unknown; decision: ExecutionReconciliation | undefined; /** Board-only source repair: never authorize a continuation of this run. */ workspaceRepairOnly?: boolean; controllerContainment?: { actionId: string; actorId: string }; }; export async function validateExecutionReconciliation(input: ExecutionReconciliationInput) { if (input.controllerContainment) { // Validation and contained-owner settlement must roll back together even // when a caller is not already inside the board route's transaction. return input.db.transaction(tx => validateExecutionReconciliationInScope({ ...input, db: tx as unknown as Db })); } return validateExecutionReconciliationInScope(input); } async function validateExecutionReconciliationInScope(input: ExecutionReconciliationInput) { const { db, companyId, issueId, agentId, decision } = input; if (!decision || decision.runId !== input.sourceRunId || (!agentId && !input.workspaceRepairOnly)) { throw conflict( "Reconcile the recorded execution and its action outcomes before continuing this task.", ); } let operatorContained = false; if (decision.controllerStopped !== undefined || decision.controllerAuthorityDigest !== undefined) { if (!decision.controllerStopped || !decision.controllerAuthorityDigest || !input.controllerContainment || input.workspaceRepairOnly) throw conflict("Original-host controller containment requires its exact board recovery action."); operatorContained = await settleOperatorContainedNativeOwner(db, { companyId, issueId, runId: decision.runId, actionId: input.controllerContainment.actionId, actorId: input.controllerContainment.actorId, authorityDigest: decision.controllerAuthorityDigest, actionOutcome: decision.actionOutcome, outcomeEvidence: decision.outcomeEvidence, }); if (!operatorContained) throw conflict("The recorded controller authority changed, is still leased, or its cleanup is unsettled. Refresh before recording containment.", { code: "native_controller_containment_stale" }); } const [run] = await db .select() .from(heartbeatRuns) .where( and( eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, decision.runId), ), ); const [task] = await db .select() .from(issues) .where(and(eq(issues.companyId, companyId), eq(issues.id, issueId))); const review = task?.status === "in_review" ? parseIssueExecutionState(task.executionState) : null; const isCurrentReviewer = review?.status === "pending" && review.currentParticipant?.type === "agent" && review.currentParticipant.agentId === run?.agentId; const retainedWorkspace = hasRequiredWorkspaceRecovery(run?.resultJson); const repairOnly = input.workspaceRepairOnly === true && retainedWorkspace; const terminalRepairSource = ["failed", "interrupted", "timed_out", "cancelled"].includes(run?.status ?? "") || (repairOnly && run?.runtimeMode === "legacy" && run.status === "succeeded"); if ( (input.workspaceRepairOnly && !retainedWorkspace) || !run || !task || (!repairOnly && (task.assigneeAgentId !== agentId || (run.agentId !== agentId && !isCurrentReviewer))) || (run.nativeIssueId ?? run.contextSnapshot?.issueId) !== issueId || !terminalRepairSource ) { throw conflict( "The recovery source or task owner changed. Inspect the current execution before continuing.", ); } if (hasWorkspaceRestoreFailure(run.resultJson) && (!decision.workspaceRepairEvidence || decision.workspaceRepairEvidence.trim().length < 20)) { throw conflict("Verify safe workspace staging or repair and record workspaceRepairEvidence before continuing this run."); } let checkLocalProcesses = !operatorContained; if (retainedWorkspace) { const leases = await db.select().from(environmentLeases).where(and( eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, run.id), )); const recovery = run.resultJson!.workspaceRestoreRecovery as { leaseIds?: unknown }; const retainedIds = recovery.leaseIds; // The terminal transaction records the exact source allocations. A release // timestamp is written before stop dispatch, so only matching provider // acknowledgements can establish that repair will not race active writes. if (!Array.isArray(retainedIds) || !retainedIds.length || !retainedIds.every(id => typeof id === "string" && leases.some(lease => lease.id === id && hasRemoteTerminationReceipt(lease))) || leases.some(lease => lease.provider === "local" ? !lease.releasedAt || lease.status === "pending_cleanup" || lease.cleanupStatus === "failed" : !hasRemoteTerminationReceipt(lease))) { throw conflict("The retained workspace environment has not confirmed that it stopped. Wait for cleanup before recording workspace repair."); } checkLocalProcesses = !operatorContained && leases.some(lease => lease.provider === "local"); } // A retained remote source's PIDs belong to its sandbox, not this server. // Its exact lease receipts above replace host-local process probes. for (const pid of checkLocalProcesses ? [ run.processPid, run.processGroupId ? -run.processGroupId : null, ] : []) { if (!pid) continue; try { process.kill(pid, 0); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") continue; throw conflict( "The previous provider's process ownership cannot be verified.", ); } throw conflict( "The previous provider is still running. Stop it before continuing.", ); } const [coordinator] = await db .select() .from(nativeRunFinalizations) .where( and( eq(nativeRunFinalizations.companyId, companyId), eq(nativeRunFinalizations.runId, run.id), ), ); if (coordinator?.leaseOwner || coordinator?.failureDetail?.successorRunId) throw conflict( "This execution still has a coordinator or a linked continuation. Inspect that run first.", ); const leases = await db .select({ id: environmentLeases.id }) .from(environmentLeases) .where( and( eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, run.id), isNull(environmentLeases.releasedAt), ), ) .limit(1); if (leases.length) throw conflict( "The previous execution environment has not finished releasing its authority.", ); if (!repairOnly) await buildExecutionContinuation({ db, companyId, issueId, agentId: agentId!, context: { previousRunId: run.id }, summary: null, exposeLowTrustRaw: false, }); return run; } /** Durable delivery marker lives on the existing source-scoped recovery action. */ export async function markExecutionReconciliation( db: Db, action: Pick< typeof issueRecoveryActions.$inferSelect, "companyId" | "id" | "evidence" | "sourceIssueId" >, decision: ExecutionReconciliation, actorId: string, deliveryOwner?: { kind: "chat_failed_run_retry"; actionId: string }, options?: { workspaceRepairOnly?: boolean; controllerContainmentOnly?: boolean }, ) { if (options?.controllerContainmentOnly && (!decision.controllerStopped || !decision.controllerAuthorityDigest || deliveryOwner)) throw conflict("Controller containment records no automatic continuation."); if (options?.workspaceRepairOnly && (deliveryOwner || !hasRequiredWorkspaceRecovery(action.evidence))) { throw conflict("Workspace repair must target the retained source without a continuation owner."); } if (deliveryOwner) { const [retry] = await db .select() .from(chatActions) .where( and( eq(chatActions.companyId, action.companyId), eq(chatActions.id, deliveryOwner.actionId), ), ); if ( deliveryOwner.kind !== "chat_failed_run_retry" || !retry || retry.kind !== "failed_run_retry" || !["issued", "processing", "processed"].includes(retry.status) || retry.payload.version !== 1 || retry.payload.failedRunId !== decision.runId || retry.payload.issueId !== action.sourceIssueId ) { throw conflict("The authorized chat retry owner is no longer valid."); } } await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionContinuation); await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionReplacement); await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionDisposition); await tx .update(nativeRunFinalizations) .set({ failureDetail: sql`coalesce(${nativeRunFinalizations.failureDetail}, '{}'::jsonb) || ${JSON.stringify({ replacementDenied: "operator_reconciled" })}::jsonb`, }) .where( and( eq(nativeRunFinalizations.companyId, action.companyId), eq(nativeRunFinalizations.runId, decision.runId), ), ); const [source] = await tx .update(issueRecoveryActions) .set({ evidence: { ...action.evidence, automaticRecovery: undefined, executionReconciliation: { ...decision, actorId, recordedAt: new Date().toISOString(), }, continuationDelivery: options?.workspaceRepairOnly || options?.controllerContainmentOnly ? "not_requested" : deliveryOwner ? "delegated" : "pending", ...(deliveryOwner ? { continuationDeliveryOwner: deliveryOwner } : {}), }, }) .where( and( eq(issueRecoveryActions.companyId, action.companyId), eq(issueRecoveryActions.id, action.id), ), ).returning({ cause: issueRecoveryActions.cause, fingerprint: issueRecoveryActions.fingerprint, kind: issueRecoveryActions.kind, evidence: issueRecoveryActions.evidence, }); if (options?.controllerContainmentOnly) { // A Retry clicked before the operator reviewed the effects must not turn // into an automatic replay after containment. Retain only that exact, // unadmitted source receipt as skipped; newer/accepted intents are untouched. await tx.update(agentWakeupRequests).set({ status: "skipped", finishedAt: new Date(), updatedAt: new Date(), error: "controller_containment_requires_new_retry" }).where(and( eq(agentWakeupRequests.companyId, action.companyId), isNull(agentWakeupRequests.runId), inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), sql`${agentWakeupRequests.payload}->>'issueId' = ${action.sourceIssueId}`, sql`${agentWakeupRequests.payload}->'taskRetry'->>'schema' = 'paperclip.task-retry.v1'`, sql`${agentWakeupRequests.payload}->'taskRetry'->>'sourceRunId' = ${decision.runId}`, )); return; } if ( !source || options?.workspaceRepairOnly || source.cause !== "legacy_execution_requires_reconciliation" || source.fingerprint !== `legacy-execution:${decision.runId}` || ["workspaceRestoreFailure", "workspaceRestoreRecovery", "systemRecovery"].some(key => key in source.evidence) ) return; const now = new Date(); const siblings = await tx.update(issueRecoveryActions).set({ status: "resolved", outcome: "cancelled", resolvedAt: now, updatedAt: now, wakePolicy: null, monitorPolicy: null, resolutionNote: "The same source execution was reconciled through another recovery action.", evidence: sql`(${issueRecoveryActions.evidence} - 'automaticRecovery') || ${JSON.stringify({ executionReconciliation: { ...decision, actorId, recordedAt: now.toISOString() }, continuationDelivery: "invalidated", coalescedIntoRecoveryActionId: action.id, })}::jsonb`, }).where(and( eq(issueRecoveryActions.companyId, action.companyId), eq(issueRecoveryActions.sourceIssueId, action.sourceIssueId), not(eq(issueRecoveryActions.id, action.id)), eq(issueRecoveryActions.kind, source.kind), eq(issueRecoveryActions.cause, source.cause), eq(issueRecoveryActions.fingerprint, source.fingerprint), sql`${issueRecoveryActions.evidence}->>'runId' = ${decision.runId}`, sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'replay' = 'blocked'`, sql`not (${issueRecoveryActions.evidence} ?| array['executionReconciliation', 'workspaceRestoreFailure', 'workspaceRestoreRecovery', 'systemRecovery'])`, )).returning({ id: issueRecoveryActions.id }); if (siblings.length) await persistActivity(tx as unknown as Db, { companyId: action.companyId, actorType: "user", actorId, action: "issue.recovery_action_resolved", entityType: "issue", entityId: action.sourceIssueId, details: { reason: "same_source_execution_reconciled", recoveryActionId: action.id, coalescedRecoveryActionIds: siblings.map(sibling => sibling.id), runId: decision.runId }, }); }); } export async function deliverReconciledExecutions( db: Db, wake: ReturnType["wakeup"], ) { const recoveryReasons = [...RECOVERY_WAKE_REASONS]; const sourceRun = alias(heartbeatRuns, "continuation_source_run"); const seenActionIds = new Set(); const seenAgentIds: Record> = { fresh: new Set(), backlog: new Set() }; const admittedByCompany = new Map(); let companyLimit = RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS; const fullCompanies = () => [...admittedByCompany].flatMap(([companyId, count]) => count >= companyLimit ? [companyId] : []); const heldSince = sql`coalesce(${issueRecoveryActions.evidence}#>>'{freshContinuation,recordedAt}', ${issueRecoveryActions.resolvedAt}::text, ${issueRecoveryActions.updatedAt}::text)::timestamptz`; const attemptedAt = sql`(${issueRecoveryActions.evidence}->>'continuationDeliveryAttemptedAt')::timestamptz`; const freshSource = () => sql`coalesce(${sourceRun.finishedAt} > ${new Date(Date.now() - RECOVERY_BACKLOG_AGE_MS).toISOString()}::timestamptz, false)`; const readPendingPage = async (kind: ContinuationClass) => { const companies = fullCompanies(); const seenAgents = seenAgentIds[kind]; const heldBound = new Date(Date.now() - RECOVERY_CONTINUATION_MAX_HOLD_MS).toISOString(); const pastBound = sql`${heldSince} <= ${heldBound}::timestamptz`; const overdueTurn = sql`case when ${pastBound} then greatest(${heldSince}, ${attemptedAt}) end`; const priority = kind === "fresh" ? [ sql`${attemptedAt} asc nulls first`, sql`${sourceRun.finishedAt} desc`, sql`${issueRecoveryActions.updatedAt} asc`, ] : [ sql`${pastBound} desc`, sql`${overdueTurn} asc nulls last`, sql`${sourceRun.finishedAt} desc nulls last`, sql`${issueRecoveryActions.updatedAt} asc`, ]; const candidates = db .select({ id: issueRecoveryActions.id, pastBound: sql`${pastBound}`.as("past_bound"), heldAt: sql`${overdueTurn}`.as("held_at"), sourceFinishedAt: sql`${sourceRun.finishedAt}`.as("source_finished_at"), attemptedAt: sql`${attemptedAt}`.as("attempted_at"), actionUpdatedAt: sql`${issueRecoveryActions.updatedAt}`.as("action_updated_at"), agentRank: sql`row_number() over ( partition by coalesce(${issueRecoveryActions.returnOwnerAgentId}::text, ${issueRecoveryActions.id}::text) order by ${sql.join(priority, sql`, `)})` .as("agent_rank"), }) .from(issueRecoveryActions) .leftJoin(sourceRun, and( eq(sourceRun.companyId, issueRecoveryActions.companyId), sql`${sourceRun.id}::text = ${issueRecoveryActions.evidence}#>>'{executionReconciliation,runId}'`, )) .where( and( eq(issueRecoveryActions.status, "resolved"), sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, seenActionIds.size ? not(inArray(issueRecoveryActions.id, [...seenActionIds])) : undefined, seenAgents.size ? sql`coalesce(${issueRecoveryActions.returnOwnerAgentId}::text, '') <> all(${sql.param([...seenAgents])}::text[])` : undefined, companies.length ? not(inArray(issueRecoveryActions.companyId, companies)) : undefined, kind === "fresh" ? freshSource() : sql`not ${freshSource()}`, kind === "fresh" ? undefined : sql`(${pastBound} or not exists (select 1 from ${heartbeatRuns} queued_recovery where queued_recovery.company_id = ${issueRecoveryActions.companyId} and queued_recovery.agent_id = ${issueRecoveryActions.returnOwnerAgentId} and queued_recovery.status = 'queued' and queued_recovery.context_snapshot->>'recoveryActionId' is distinct from ${issueRecoveryActions.id}::text and queued_recovery.context_snapshot->>'wakeReason' in (${sql.join(recoveryReasons.map(reason => sql`${reason}`), sql`, `)})))`, ), ) .as("continuation_candidates"); const rows = await db .select({ action: issueRecoveryActions }) .from(candidates) .innerJoin(issueRecoveryActions, eq(issueRecoveryActions.id, candidates.id)) .where(eq(candidates.agentRank, 1)) .orderBy(...(kind === "fresh" ? [ sql`${candidates.attemptedAt} asc nulls first`, sql`${candidates.sourceFinishedAt} desc`, sql`${candidates.actionUpdatedAt} asc`, ] : [ sql`${candidates.pastBound} desc`, sql`${candidates.heldAt} asc nulls last`, sql`${candidates.sourceFinishedAt} desc nulls last`, sql`${candidates.actionUpdatedAt} asc`, ])) .limit(25); return rows.map(row => row.action); }; const deferRefusedContinuation = (action: typeof issueRecoveryActions.$inferSelect) => db.update(issueRecoveryActions) .set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify( { continuationDeliveryAttemptedAt: new Date().toISOString() })}::jsonb` }) .where(and( eq(issueRecoveryActions.companyId, action.companyId), eq(issueRecoveryActions.id, action.id), eq(issueRecoveryActions.status, "resolved"), sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, )); const agentHasRoom = async (action: typeof issueRecoveryActions.$inferSelect, agentId: string, kind: ContinuationClass) => { if ((admittedByCompany.get(action.companyId) ?? 0) >= companyLimit) return false; if (kind === "fresh") return true; const [agent] = await db.select().from(agents).where(and(eq(agents.companyId, action.companyId), eq(agents.id, agentId))); if (!agent) return true; const [queued] = await db.select({ count: sql`count(*)::int` }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, action.companyId), eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued"))); if (Number(queued?.count ?? 0) < parseHeartbeatPolicy(agent).maxConcurrentRuns) return true; return Date.now() - recoveryContinuationHeldSince(action) >= RECOVERY_CONTINUATION_MAX_HOLD_MS; }; const cap = RECOVERY_CONTINUATIONS_PER_COMPANY_PER_PASS; const phases: Array<[ContinuationClass, number]> = [ ["fresh", cap - RECOVERY_CONTINUATION_BACKLOG_SHARE_PER_PASS], ["backlog", cap], ["fresh", cap]]; for (let page = 0; page < RECOVERY_CONTINUATION_PAGES_PER_PASS * phases.length; page += 1) { const phase = Math.floor(page / RECOVERY_CONTINUATION_PAGES_PER_PASS); const [kind, limit] = phases[phase]!; companyLimit = limit; const seenAgents = seenAgentIds[kind]; const pending = await readPendingPage(kind); if (!pending.length) { page = (phase + 1) * RECOVERY_CONTINUATION_PAGES_PER_PASS - 1; continue; } for (const action of pending) { if ((admittedByCompany.get(action.companyId) ?? 0) >= companyLimit) continue; seenActionIds.add(action.id); if (action.returnOwnerAgentId && seenAgents.has(action.returnOwnerAgentId)) continue; try { const decision = action.evidence.executionReconciliation as ExecutionReconciliation | undefined; if (!decision || !action.returnOwnerAgentId) continue; const pendingDecision = and( eq(issueRecoveryActions.companyId, action.companyId), eq(issueRecoveryActions.id, action.id), eq(issueRecoveryActions.status, "resolved"), sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`, sql`${issueRecoveryActions.evidence}->'executionReconciliation' = ${JSON.stringify(decision)}::jsonb`, ); const [task] = await db .select() .from(issues) .where( and( eq(issues.companyId, action.companyId), eq(issues.id, action.sourceIssueId), ), ); if ( !task || task.assigneeAgentId !== action.returnOwnerAgentId || ["done", "cancelled"].includes(task.status) ) { await db .update(issueRecoveryActions) .set({ evidence: sql`${issueRecoveryActions.evidence} || '{"continuationDelivery":"invalidated"}'::jsonb`, }) .where(pendingDecision); await settleCoalescedFreshContinuationHolds(db, { companyId: action.companyId, primaryId: action.id, delivery: "invalidated" }); continue; } // A fresh continuation is the task's next run only while nothing else // started one: a person's Retry or saved input that ran since it was // recorded already continues the task, so it is never a second run. const fresh = action.evidence.freshContinuation as { recordedAt?: unknown } | undefined; if (fresh && typeof fresh.recordedAt === "string") { const [newer] = await db.select({ id: heartbeatRuns.id, contextSnapshot: heartbeatRuns.contextSnapshot }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, action.companyId), sql`${heartbeatRuns.id}::text <> ${decision.runId}`, or(eq(heartbeatRuns.nativeIssueId, task.id), eq(heartbeatRuns.issueId, task.id), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${task.id}::text`), sql`${heartbeatRuns.createdAt} >= ${fresh.recordedAt}::timestamptz`)).limit(1); if (newer) { await db.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify( { continuationDelivery: "delivered", continuationRunId: newer.id, continuationSupersededByRun: true })}::jsonb` }) .where(pendingDecision); await settleCoalescedFreshContinuationHolds(db, { companyId: action.companyId, primaryId: action.id, delivery: "delivered", continuationRunId: newer.id }); if (newer.contextSnapshot?.recoveryActionId === action.id) await db.transaction(async tx => { await tx.execute(sql`select id from issues where company_id = ${action.companyId} and id = ${task.id} for update`); await foldWaitingRetriesIntoContinuation(tx as unknown as Db, { companyId: action.companyId, issueId: task.id, runId: newer.id, sourceRunId: decision.runId }); }); continue; } } if (!await agentHasRoom(action, action.returnOwnerAgentId, kind)) { seenAgents.add(action.returnOwnerAgentId); continue; } const run = await wake(action.returnOwnerAgentId, { source: "automation", triggerDetail: "system", reason: "issue_recovery_action_restored", idempotencyKey: `execution-reconciliation:${action.id}`, payload: { issueId: task.id, recoveryActionId: action.id }, requestedByActorType: "system", requestedByActorId: "execution-recovery", contextSnapshot: { issueId: task.id, taskId: task.id, recoveryActionId: action.id, previousRunId: decision.runId, retryOfRunId: decision.runId, forceFreshSession: true, wakeReason: "issue_recovery_action_restored", source: "execution.reconciled", ...((action.evidence.freshContinuation as { isolation?: unknown } | undefined)?.isolation === "fresh_isolated_workspace" ? { freshContinuationIsolation: action.id } : {}), }, }); seenAgents.add(action.returnOwnerAgentId); if (!run) { await deferRefusedContinuation(action); continue; } admittedByCompany.set(action.companyId, (admittedByCompany.get(action.companyId) ?? 0) + 1); await db.transaction(async (tx) => { await tx .update(heartbeatRuns) .set({ retryOfRunId: decision.runId }) .where( and( eq(heartbeatRuns.companyId, action.companyId), eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.agentId, action.returnOwnerAgentId!), sql`${heartbeatRuns.contextSnapshot}->>'recoveryActionId' = ${action.id}`, sql`${heartbeatRuns.contextSnapshot}->>'previousRunId' = ${decision.runId}`, ), ); await tx .update(issueRecoveryActions) .set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify( { continuationDelivery: "delivered", continuationRunId: run.id, }, )}::jsonb`, }) .where(pendingDecision); await settleCoalescedFreshContinuationHolds(tx as unknown as Db, { companyId: action.companyId, primaryId: action.id, delivery: "delivered", continuationRunId: run.id }); if (fresh) await foldWaitingRetriesIntoContinuation(tx as unknown as Db, { companyId: action.companyId, issueId: task.id, runId: run.id, sourceRunId: decision.runId }); }); } catch { if (action.returnOwnerAgentId) seenAgents.add(action.returnOwnerAgentId); await deferRefusedContinuation(action).catch(() => undefined); logger.warn( { recoveryActionId: action.id }, "Reconciled execution continuation remains pending for retry", ); } } } } type ContinuationClass = "fresh" | "backlog"; const supersedingWorkspaceHold = alias(issueRecoveryActions, "superseding_workspace_hold"); function supersedingNativeWorkspaceHold(db: Db, input: { companyId: string | typeof heartbeatRuns.companyId; issueId: string | typeof issueRecoveryActions.sourceIssueId; runId: string | typeof heartbeatRuns.id; }) { return db.select({ id: supersedingWorkspaceHold.id }).from(supersedingWorkspaceHold).where(and( eq(supersedingWorkspaceHold.companyId, input.companyId), eq(supersedingWorkspaceHold.sourceIssueId, input.issueId), eq(supersedingWorkspaceHold.status, "resolved"), eq(supersedingWorkspaceHold.cause, "native_continuation_requires_reconciliation"), sql`${supersedingWorkspaceHold.evidence}->>'runId' = ${input.runId}::text`, sql`${supersedingWorkspaceHold.evidence}->'automaticRecovery'->>'replay' = 'blocked'`, sql`${supersedingWorkspaceHold.evidence}->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1'`, )); } export function unresolvedExecutionDispositions(db: Db, limit: number) { return db .select({ action: issueRecoveryActions }) .from(issueRecoveryActions) .innerJoin( heartbeatRuns, and( eq(heartbeatRuns.companyId, issueRecoveryActions.companyId), eq(heartbeatRuns.id, canonicalExecutionUuid(sql`${issueRecoveryActions.evidence}->>'runId'`)), or( eq(heartbeatRuns.nativeIssueId, issueRecoveryActions.sourceIssueId), and(isNull(heartbeatRuns.nativeIssueId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueRecoveryActions.sourceIssueId}::text`), ), ), ) .leftJoin( nativeRunFinalizations, and( eq(nativeRunFinalizations.companyId, heartbeatRuns.companyId), eq(nativeRunFinalizations.runId, heartbeatRuns.id), ), ) .where( and( not(conversationRecoveryActionPredicate()!), sql`not (${issueRecoveryActions.evidence} ? 'systemRecovery')`, inArray(issueRecoveryActions.status, ["active", "escalated"]), eq(issueRecoveryActions.kind, "active_run_watchdog"), // A fresh continuation that backed off is checked again when due. freshContinuationCheckDue(new Date()), or(inArray(issueRecoveryActions.cause, [...EXECUTION_RECONCILIATION_CAUSES]), and( eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.nativePhase, "terminal_failure"), eq(nativeRunFinalizations.phase, "terminal_failure"), isNull(nativeRunFinalizations.nextAttemptAt), sql`${issueRecoveryActions.wakePolicy}->>'kind' = 'resume_native_run'`, sql`${issueRecoveryActions.wakePolicy}->>'runId' = ${heartbeatRuns.id}::text`, exists(supersedingNativeWorkspaceHold(db, { companyId: heartbeatRuns.companyId, issueId: issueRecoveryActions.sourceIssueId, runId: heartbeatRuns.id })), )), inArray(heartbeatRuns.status, [ "failed", "timed_out", "interrupted", "cancelled", ]), isNull(nativeRunFinalizations.leaseOwner), isNull(nativeRunFinalizations.resultId), or( isNull(nativeRunFinalizations.runId), eq(nativeRunFinalizations.phase, "terminal_failure"), ), sql`coalesce(${nativeRunFinalizations.failureDetail}->>'successorRunId', '') = ''`, sql`(${heartbeatRuns.runtimeMode} <> 'native' or coalesce(${nativeRunFinalizations.failureCode}, '') <> 'native_provider_terminal_failed' or coalesce(${nativeRunFinalizations.failureDetail}->>'replacementDenied', '') <> '')`, ), ) .limit(limit); } export function obsoleteExecutionConversationHold() { return and( conversationRecoveryActionPredicate(), sql`not (${issueRecoveryActions.evidence} ? 'systemRecovery')`, or( inArray(issueRecoveryActions.status, ["active", "escalated"]), sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'replay' = 'blocked'`, ), ); } /** * Failed execution is a system responsibility, not a user questionnaire. After * automatic recovery is ruled out, preserve evidence and stop without replay. * This is NOT evidence that an external action succeeded or never happened. * The resolved record retains a dispatch hold until actual evidence clears it. */ export async function settleUnrecoverableExecutions( db: Db, now = new Date(), options: { failpoint?: (phase: "persisted") => void } = {}, ) { // Fold obsolete conversation holds without waking historical work on upgrade. // Keep their evidence and record the policy change in the task's activity log. const obsoleteConversationHold = obsoleteExecutionConversationHold(); await db.transaction(async tx => { const foldable = await tx.select().from(issueRecoveryActions).where(obsoleteConversationHold) .limit(25).for("update", { skipLocked: true }); for (const candidate of foldable) { if (await getConversationOwnershipBlocker(tx as unknown as Db, candidate.companyId, candidate.sourceIssueId)) continue; await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: candidate.companyId, issueId: candidate.sourceIssueId }); const [action] = await tx.update(issueRecoveryActions).set({ status: "resolved", outcome: "cancelled", resolvedAt: now, updatedAt: now, nextAction: "Automatic attempts stopped. Send a new message to continue the conversation.", resolutionNote: "Conversation continuation does not replay prior tool calls.", wakePolicy: null, monitorPolicy: null, evidence: sql`case when ${issueRecoveryActions.evidence} ? 'automaticRecovery' then jsonb_set(${issueRecoveryActions.evidence}, '{automaticRecovery,replay}', '"conversation_continuation"'::jsonb) else ${issueRecoveryActions.evidence} end`, }).where(and(obsoleteConversationHold, eq(issueRecoveryActions.id, candidate.id))).returning(); if (!action) continue; await persistActivity(tx as unknown as Db, { companyId: action.companyId, actorType: "system", actorId: "execution-recovery", action: "issue.execution_recovery_settled", entityType: "issue", entityId: action.sourceIssueId, details: { recoveryActionId: action.id, outcome: "cancelled", continuation: "conversation" }, }); } }); // Filter eligibility before applying the batch limit. A queue of sessions // awaiting replacement must not starve settled incidents behind it. const candidates = await unresolvedExecutionDispositions(db, 25); for (const { action: candidate } of candidates) { const runId = candidate.evidence.runId; if (typeof runId !== "string") continue; const goneControllerBootId = await provenGoneLegacyControllerBootId(db, candidate.companyId, runId).catch(() => null); try { await db.transaction(async (tx) => { await tx.execute( sql`select set_config('statement_timeout', '15000', true), set_config('lock_timeout', '1000', true)`, ); // Same issue -> coordinator -> run ordering as replacement/finalization. const [task] = await tx .select() .from(issues) .where( and( eq(issues.companyId, candidate.companyId), eq(issues.id, candidate.sourceIssueId), ), ) .for("update"); const [coordinator] = await tx .select() .from(nativeRunFinalizations) .where( and( eq(nativeRunFinalizations.companyId, candidate.companyId), eq(nativeRunFinalizations.runId, runId), ), ) .for("update"); const [run] = await tx .select() .from(heartbeatRuns) .where( and( eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.id, runId), ), ) .for("update"); const [action] = await tx .select() .from(issueRecoveryActions) .where(eq(issueRecoveryActions.id, candidate.id)) .for("update"); if ( !task || !run || !action || action.evidence.runId !== runId || !["active", "escalated"].includes(action.status) || (run.nativeIssueId ?? run.contextSnapshot?.issueId) !== task.id || !["failed", "timed_out", "interrupted", "cancelled"].includes( run.status, ) ) return; if (!EXECUTION_RECONCILIATION_CAUSES.includes(action.cause as (typeof EXECUTION_RECONCILIATION_CAUSES)[number])) { if (run.runtimeMode !== "native" || run.nativePhase !== "terminal_failure" || coordinator?.phase !== "terminal_failure" || coordinator.leaseOwner || coordinator.resultId || coordinator.nextAttemptAt || coordinator.failureDetail?.successorRunId || action.wakePolicy?.kind !== "resume_native_run" || action.wakePolicy.runId !== run.id) return; const [hold] = await supersedingNativeWorkspaceHold(tx as unknown as Db, { companyId: run.companyId, issueId: task.id, runId: run.id, }).limit(1).for("update"); if (!hold) return; await notifyExecutionRunWork(tx, { companyId: run.companyId }); const note = "Exact session recovery stopped. The recorded workspace recovery holds this execution without replaying its actions."; await tx.update(issueRecoveryActions).set({ status: "cancelled", outcome: "cancelled", resolvedAt: now, updatedAt: now, wakePolicy: null, monitorPolicy: null, nextAction: note, resolutionNote: note, evidence: { ...action.evidence, supersedingRecoveryActionId: hold.id }, }).where(eq(issueRecoveryActions.id, action.id)); await persistActivity(tx as unknown as Db, { companyId: run.companyId, actorType: "system", actorId: "execution-recovery", action: "issue.execution_recovery_settled", entityType: "issue", entityId: task.id, runId: run.id, details: { recoveryActionId: action.id, outcome: "cancelled", supersedingRecoveryActionId: hold.id }, }); return; } // A legacy executor still holding its controller lease may be unwinding // a Stop and can prove that no adapter started. Let it settle first, // unless its recorded process is proven gone (a drained server): then // expire that exact lease so every later preparation check sees it. if (run.runtimeMode === "legacy" && run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > now) { if (goneControllerBootId === null || run.controllerBootId !== goneControllerBootId) return; await tx.update(heartbeatRuns).set({ controllerLeaseExpiresAt: now }).where(and( eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.controllerBootId, goneControllerBootId))); run.controllerLeaseExpiresAt = now; } // Adoption transfers maintenance to the retained-workspace owner. // This compatibility resolver must not close its hold or relabel it // as a user repair merely because old return-owner metadata is absent. if (readSystemRecovery(action.evidence)) return; // Give durable native recovery its chance; never preempt a resume, // replacement, result finalizer, or still-owned execution. if ( coordinator?.leaseOwner || coordinator?.resultId || coordinator?.failureDetail?.successorRunId || (coordinator && coordinator.phase !== "terminal_failure") || (run.runtimeMode === "native" && coordinator?.failureCode === "native_provider_terminal_failed" && !coordinator.failureDetail?.replacementDenied) ) return; const current = !isSupersededConversationRun(task, run) && action.returnOwnerAgentId !== null && task.assigneeAgentId === action.returnOwnerAgentId && !["done", "cancelled"].includes(task.status) && (!task.executionRunId || task.executionRunId === run.id) && (!task.checkoutRunId || task.checkoutRunId === run.id); // Exact recovery is ruled out. Unless the cause is the person's, the // task continues in a fresh run from its last saved work instead of // waiting for a person. // A preparation that never started provider work keeps its own replay // lane, which takes over the settled hold; the held sweep follows it. if (current && !isInterruptedNativePreparationCandidate(run)) { const fresh = await continueFreshInTransaction(tx as unknown as Db, { companyId: task.companyId, runId: run.id, trigger: "recovery_refused", now }); if (["continued", "already_continued", "successor_exists"].includes(fresh.kind)) return; if (fresh.kind === "backoff" || fresh.kind === "exact_recovery_pending") { await recordFreshContinuationCheck(tx as unknown as Db, { companyId: task.companyId, actionId: action.id }, fresh, now); return; } } const note = executionRecoverySettlementNote({ current, cause: action.cause, workspaceRestoreFailed: hasWorkspaceRestoreFailure(run.resultJson), trigger: action.evidence.trigger }); await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: task.companyId, issueId: task.id }); let nativeFailureBlock = action.evidence.nativeFailureBlock; let nativeBootstrapFailureBlock = action.evidence.nativeBootstrapFailureBlock; let legacyCancellationBlock = action.evidence.legacyCancellationBlock; if (current) { await notifyNativeStatusReassessment(tx, { companyId: task.companyId, issueId: task.id }); if (task.status !== "blocked") await notifyChatResponseAvailability(tx, { companyId: task.companyId, issueId: task.id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyRoutineExecutionStatus(tx, task.id, task.companyId); const [projected] = await tx .update(issues) .set({ status: "blocked", executionRunId: null, checkoutRunId: null, updatedAt: now, }) .where(eq(issues.id, task.id)).returning(); // Only a transition owned by this failure grants a recovery receipt. // An already-blocked task may have a separate human/dependency hold. if (task.status !== "blocked" && run.runtimeMode === "native") { nativeFailureBlock = { runId: run.id, statusVersion: projected!.statusVersion }; } if (task.status !== "blocked" && run.runtimeMode === "legacy" && run.status === "cancelled") { legacyCancellationBlock = { runId: run.id, statusVersion: projected!.statusVersion, previousStatus: task.status }; } if (task.status !== "blocked" && isInterruptedNativePreparationCandidate(run)) { nativeBootstrapFailureBlock = { runId: run.id, statusVersion: projected!.statusVersion, previousStatus: task.status }; } } await tx .update(issueRecoveryActions) .set({ status: "resolved", outcome: current ? "blocked" : "cancelled", resolvedAt: now, updatedAt: now, nextAction: note, resolutionNote: note, wakePolicy: null, monitorPolicy: null, evidence: { ...action.evidence, ...(nativeFailureBlock ? { nativeFailureBlock } : {}), ...(nativeBootstrapFailureBlock ? { nativeBootstrapFailureBlock } : {}), ...(legacyCancellationBlock ? { legacyCancellationBlock } : {}), automaticRecovery: { policy: "preserve_without_replay_v1", runId: run.id, replay: "blocked", actionOutcome: "unknown", recordedAt: now.toISOString(), }, }, }) .where(eq(issueRecoveryActions.id, action.id)); await persistActivity(tx as unknown as Db, { companyId: run.companyId, actorType: "system", actorId: "execution-recovery", action: "issue.execution_recovery_settled", entityType: "issue", entityId: task.id, runId: run.id, details: { recoveryActionId: action.id, outcome: current ? "blocked" : "cancelled", replay: "not_authorized", }, }); await notifyExecutionRunWork(tx, { companyId: run.companyId }); await tx .update(heartbeatRuns) .set({ executionStatusDeliveryId: randomUUID() }) .where(eq(heartbeatRuns.id, run.id)); await appendHeartbeatRunEvent(tx as unknown as Db, { companyId: run.companyId, agentId: run.agentId, runId: run.id, eventType: "lifecycle", stream: "system", level: "warn", message: note, payload: { recoveryActionId: action.id, cause: action.cause, automaticRecovery: "preserve_without_replay_v1", replay: "blocked", }, }); options.failpoint?.("persisted"); }); } catch (err) { if (options.failpoint) throw err; logger.warn( { err, recoveryActionId: candidate.id }, "Automatic recovery disposition remains pending", ); } } // Tasks already held by settled records (including those older releases // saved) continue from their saved work on this worker too. await continueHeldExecutions(db, now); }