import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; import { authorizationService } from "../authorization.js"; import { assertTaskRetryConversationAllowed, readTaskRetryIntent, retireSupersededTaskRetry } from "../task-retry.js"; import { insertIssueComment } from "../issue-comment-writes.js"; import { withExternalAdmissionGuard, assertNoExternalOverlap } from "../../modules/external-agents/index.js"; import { isAgentAwaitingSetup } from "../../modules/agent-lifecycle/index.js"; import { normalizeAgentNameKey } from "./retries.js"; import { publishActiveDotComment } from "../dot-assignment-follow-up.js"; import { publishActiveMuseComment } from "../muse-assignment-follow-up.js"; import { type WakeupOptions, mergeCoalescedContextSnapshot, enrichWakeContextSnapshot, hasInteractionContinuationWakeContext, isInteractionResolutionWakePayload, } from "./run-preparation.js"; import { resolveExecutionWorkspaceReuseRequestForIssue } from "./workspaces.js"; import { CHAT_COMPLETION_WAKE_REASON, isCompletedOnboardingHandoffWake, } from "../chat-completion-delivery.js"; import { compareCents, hasWorkspaceRestoreFailure, } from "@paperclipai/shared"; import { nonIdleSlackIssueCondition } from "../slack-conversation-state.js"; import { readQueuedInteractionResponse } from "../queued-interaction-response.js"; import { isConversation, isConversationExecutionWake, isWaitingConversation, } from "../agent-conversations.js"; import { legacyControllerClaim } from "../legacy-controller-lease.js"; import { admitExplicitNativeContinuation, undeliveredLegacyUserCommentIds, } from "../explicit-native-continuation.js"; import { canRetryStoppedRun } from "../cancelled-native-startup.js"; import { getExecutionBlocker, hasPendingRunCleanup } from "../execution-blocker.js"; import { validateNativeWorkspaceReconstruction } from "../native-runtime/native-workspace-reconstruction.js"; import { isConversationAdapter } from "../conversation-continuation.js"; import { recordExecutionWait } from "../execution-wait.js"; import { getNativeReviewAssignment, readNativeReviewAssignmentContext, } from "../native-runtime/native-review-participant.js"; import { claimQueuedNativeReviewRun } from "../native-runtime/native-review-dispatch.js"; import { adapterExecutionControls } from "../adapter-execution-control.js"; import { executionFailureRetryCount } from "../execution-recovery-attempt.js"; import { slackMentionAllowsRunStart, assertDurableChatWakeupReceipt, assertDurableChatWakeupRequest, authorizeFailedChatRunRetryWake, FailedChatRunRetryAuthorizationError, unadmittedChatWakeupCondition, } from "../durable-chat-wakeup.js"; import { agentService } from "../agents.js"; import { normalizeLegacyRunnerProvider } from "@paperclipai/adapter-utils"; import { randomUUID } from "node:crypto"; import { and, asc, desc, eq, gt, gte, inArray, isNull, lt, lte, ne, notInArray, or, sql, } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { AGENT_DEFAULT_MAX_CONCURRENT_RUNS } from "@paperclipai/shared"; import { agents, caseHandoffs, cases, agentWakeupRequests, activityLog, chatConversations, chatEndpoints, companies, costEvents, executionWorkspaces, heartbeatRunEvents, heartbeatRuns, issueComments, issueRecoveryActions, issueRelations, issues, } from "@paperclipai/db"; import { conflict, HttpError, notFound, } from "../../errors.js"; import { logger } from "../../middleware/logger.js"; import { publishLiveEvent } from "../live-events.js"; import { allocateHeartbeatRunEventSeq } from "../heartbeat-run-events.js"; import { queuedCommentIdsFromWakePayload, queuedCommentIdsFromRunContext, withQueuedCommentIdsInWakePayload, withQueuedCommentIdsInRunContext, } from "../issue-queued-comment-queue.js"; import { parseObject, asBoolean, asNumber, } from "../../adapters/utils.js"; import { isExternalChatWaitAuthorizationContention } from "../native-runtime/chat-attachment-reuse.js"; import { emitAgentTaskRun } from "../agent-task-run-telemetry.js"; import { CHAT_CONTROL_RECOVERY_ADMISSION_KEY, CHAT_CONTROL_RECOVERY_STOP_CODE, CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE, chatControlRecoveryAdmission, readChatControlRecoveryAdmission, readChatControlNativeParent, readChatControlRecoveryStop, } from "../chat-control-recovery-stop.js"; import { ISSUE_NEW_INPUT_ACTIVITY_ACTIONS, ISSUE_PROGRESS_ACTIVITY_ACTIONS, ISSUE_REWAKE_LOOKBACK_MS, ISSUE_REWAKE_RUN_SAMPLE_LIMIT, evaluateIssueRewakeThrottle, isThrottleCandidateIssueRewake, } from "../issue-rewake-throttle.js"; import { logActivity } from "../activity-log.js"; import { ISSUE_BLOCKERS_RESOLVED_WAKE_REASON } from "../issue-dependency-wakeups.js"; import { ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS, isVerifiedIssueTreeControlInteractionWake, } from "../issue-tree-control.js"; import { buildExecutionWorkspaceAdapterConfig, isUnrunnableWorktreeCombo, parseIssueExecutionWorkspaceSettings, resolveEffectiveWorkspaceStrategyType, resolveExecutionWorkspaceMode, WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE, WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION, } from "../execution-workspace-policy.js"; import { readContinuationAttempt } from "../recovery/index.js"; import { allowsIssueInteractionWake, wakeChatConversationsConflict } from "../../modules/run-dispatch/index.js"; import { withAgentStartLock } from "../agent-start-lock.js"; import { evaluateAgentInvokability, shouldCancelRunsForNonInvokableAgent, type AgentOrgRow, } from "../agent-invokability.js"; import { isHeartbeatWakeOnDemandEnabled } from "../heartbeat-policy.js"; import { isUuidLike } from "@paperclipai/shared"; import { retryChatControlAdmission } from "../chat-control-admission-retry.js"; import type { HeartbeatServiceOptions } from "../heartbeat.js"; import type { createHeartbeatRunState } from "./run-state.js"; import type { createHeartbeatRunPreparation } from "./run-preparation.js"; import type { HeartbeatRetryDependencies } from "./retries.js"; import type { HeartbeatRecoveryDependencies, createHeartbeatRecovery } from "./recovery.js"; import type { issueService } from "../issues.js"; import type { issueTreeControlService } from "../issue-tree-control.js"; import type { budgetService } from "../budgets.js"; import type { instanceSettingsService } from "../instance-settings.js"; import type { createRunDispatch } from "../../modules/run-dispatch/index.js"; import type { PostCommitEffect as WakeQueuePostCommitEffect, createWakeQueue } from "../../modules/wake-queue/index.js"; type HeartbeatRun = typeof heartbeatRuns.$inferSelect; type HeartbeatRunState = ReturnType; type RunPreparation = ReturnType; /** Queue policy uses the service's lifecycle effects and process-wide ownership sets. */ export interface HeartbeatQueueDependencies extends Pick, Pick, Pick, Pick { sweepPendingCleanupLeases: ReturnType["sweepPendingCleanupLeases"]; options: Pick; issuesSvc: Pick, "getDependencyReadiness" | "listDependencyReadiness">; treeControlSvc: Pick, "getActivePauseHoldGate">; budgets: Pick, "getInvocationBlock">; instanceSettings: Pick, "getExperimental">; runDispatch: Pick, "cancelStaleQueuedRun">; wakeQueue: Pick, "createAdmissionTransactionScope" | "admitWakeBehindIssueExecution">; activeRunExecutions: Set; activeRunExecutionPromises: Set>; activeWakeupPromises: Set>; liveRunExecutions: { has(id: string): boolean }; isHeartbeatRunTerminalStatus: (status: string | null | undefined) => boolean; isSameTaskScope: (left: string | null, right: string | null) => boolean; runTaskKey: (run: HeartbeatRun) => string | null; filterZombieCoalesceTarget: (run: T | null, live: { has(id: string): boolean }) => T | null; getSchedulingSuppression: () => Promise<{ suppressed: boolean; reason: "worktree_instance" | "database_restore_in_progress" | "task_drain" | null }>; canStartNewExternalBotWork: (agent: typeof agents.$inferSelect) => Promise; executeRun: (runId: string) => Promise; releaseIssueExecutionAndPromote: (run: Pick, options?: { suppressImmediateRecovery?: boolean; resumeStrandedQueue?: boolean; deferredPostCommitEffects?: WakeQueuePostCommitEffect[] }) => Promise; applyWakeQueuePostCommitEffects: (effects: WakeQueuePostCommitEffect[]) => Promise; publishRunLifecyclePluginEvent: (run: HeartbeatRun) => void; cancelRunInternal: (runId: string, reason?: string, options?: { errorCode?: string }) => Promise; cancelActiveForAgentInternal: (agentId: string, reason: string) => Promise; resumeExecutionWaitComments: () => Promise; resumeQueuedCommentInterrupt: (companyId: string, queueId: string) => Promise; resumeSavedLegacyComments: (companyId: string, queueId: string) => Promise; formatIssueIdentifierLink: (identifier: string | null, fallback: string) => string; } const HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT = AGENT_DEFAULT_MAX_CONCURRENT_RUNS; const HEARTBEAT_MAX_CONCURRENT_RUNS_MIN = 1; const HEARTBEAT_MAX_CONCURRENT_RUNS_MAX = 50; export const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext"; export const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = [ "queued", "running", "scheduled_retry", ] as const; const TIMER_ACTIONABLE_ISSUE_STATUSES = ["todo", "in_progress"] as const; export class ChatControlRecoveryUnresolvedError extends Error { constructor() { super( "Run admission could not acquire its database locks after bounded retries. No provider work started. Review database contention and send a fresh request; this attempt will not automatically retry.", ); } } const RUNNING_ISSUE_WAKE_REASONS_REQUIRING_FOLLOWUP = new Set([ CHAT_COMPLETION_WAKE_REASON, "approval_approved", ISSUE_BLOCKERS_RESOLVED_WAKE_REASON, "issue_recovery_action_restored", ]); function normalizeMaxConcurrentRuns(value: unknown) { const parsed = Math.floor( asNumber(value, HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT), ); if (!Number.isFinite(parsed)) return HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT; return Math.max( HEARTBEAT_MAX_CONCURRENT_RUNS_MIN, Math.min(HEARTBEAT_MAX_CONCURRENT_RUNS_MAX, parsed), ); } export function shouldRequireIssueCommentForWake( contextSnapshot: Record | null | undefined, ) { if (contextSnapshot?.skipIssueComment === true) return false; const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason); return ( wakeReason === "issue_assigned" || wakeReason === "execution_review_requested" || wakeReason === "execution_approval_requested" || wakeReason === "execution_changes_requested" ); } async function listUnresolvedBlockerSummaries( dbOrTx: Pick, companyId: string, issueId: string, unresolvedBlockerIssueIds: string[], ) { const ids = [...new Set(unresolvedBlockerIssueIds.filter(Boolean))]; if (ids.length === 0) return []; return dbOrTx .select({ id: issues.id, identifier: issues.identifier, title: issues.title, status: issues.status, priority: issues.priority, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, }) .from(issueRelations) .innerJoin(issues, eq(issueRelations.issueId, issues.id)) .where( and( eq(issueRelations.companyId, companyId), eq(issueRelations.type, "blocks"), eq(issueRelations.relatedIssueId, issueId), inArray(issues.id, ids), ), ) .orderBy(asc(issues.title)); } export function shouldDeferFollowupWakeForSameIssue(input: { activeRunStatus: string | null | undefined; isSameExecutionAgent: boolean; wakeCommentId: string | null | undefined; forceFreshSession: boolean; }) { // A comment follow-up or explicit fresh-session wake needs a new run boundary. if (!input.isSameExecutionAgent) return false; if (input.activeRunStatus !== "running") return false; if (input.wakeCommentId) return true; if (input.forceFreshSession) return true; return false; } export function shouldQueueFollowupForRunningIssueWake(input: { contextSnapshot: Record | null | undefined; wakeCommentId: string | null; }) { if (input.wakeCommentId) return true; // A structured interaction response is new input just like a comment. It // must run after the turn that created the interaction instead of being // merged into that still-running turn. if ( readNonEmptyString(input.contextSnapshot?.interactionId) && readNonEmptyString(input.contextSnapshot?.interactionStatus) ) { return true; } const wakeReason = readNonEmptyString(input.contextSnapshot?.wakeReason); if (wakeReason === "issue_children_completed" && ( input.contextSnapshot?.onboardingCompletion === true || (input.contextSnapshot?.statusDecisionSource === "native_status_decision" && readNonEmptyString(input.contextSnapshot?.nativeChildCompletionDecisionId)) )) return true; return Boolean( wakeReason && RUNNING_ISSUE_WAKE_REASONS_REQUIRING_FOLLOWUP.has(wakeReason), ); } function readNonEmptyString(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value : null; } /** Queue admission and dispatch share the service lifecycle callbacks and execution ownership. */ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDependencies) { const { setRunStatus, setWakeupStatus, appendRunEvent, releaseIssueExecutionAndPromote, applyWakeQueuePostCommitEffects, issuesSvc, options, publishRunLifecyclePluginEvent, finalizeAgentStatus, getAgent, cancelRunInternal, toAgentOrgRow, getAgentInvokability, budgets, activeRunExecutions, treeControlSvc, runDispatch, applyRunDispatchPostCommitEffects, resolveResponsibleUserIdForRun, getIssueExecutionContext, isHeartbeatRunTerminalStatus, liveRunExecutions, getSchedulingSuppression, resumeExecutionWaitComments, getWorktreeExecutionCutoff, resumeQueuedCommentInterrupt, resumeSavedLegacyComments, cancelActiveForAgentInternal, listCompanyAgentOrgRows, executeRun, canStartNewExternalBotWork, activeRunExecutionPromises, activeWakeupPromises, instanceSettings, getRun, sweepPendingCleanupLeases, resolveExplicitResumeSessionOverride, resolveSessionBeforeForWakeup, hasResolvablePriorSessionWorkspaceForWake, getRoutineEnvForExecutionIssue, resolveResponsibleUserIdForRunSeed, formatIssueIdentifierLink, wakeQueue, isSameTaskScope, runTaskKey, filterZombieCoalesceTarget, } = dependencies; function parseHeartbeatPolicy(agent: typeof agents.$inferSelect) { const runtimeConfig = parseObject(agent.runtimeConfig); const heartbeat = parseObject(runtimeConfig.heartbeat); return { enabled: asBoolean(heartbeat.enabled, false), intervalSec: Math.max(0, asNumber(heartbeat.intervalSec, 0)), wakeOnDemand: isHeartbeatWakeOnDemandEnabled(agent), // A Dot binding has one external turn. Competing assignments must retain // their queue position instead of claiming a second run that cannot bind. maxConcurrentRuns: agent.adapterType === "paperclip_runner" && ["openai_dot", "grok_bot", "muse"].includes(String(parseObject(agent.adapterConfig).provider)) ? 1 : normalizeMaxConcurrentRuns(heartbeat.maxConcurrentRuns), skipTimerWhenNoActionableWork: asBoolean( heartbeat.skipTimerWhenNoActionableWork ?? heartbeat.requireActionableTimerWork ?? heartbeat.issueOnlyTimer, false, ), maxDailyRuns: normalizeOptionalNonNegativeInteger( heartbeat.maxDailyRuns ?? heartbeat.dailyRunLimit ?? heartbeat.dailyRunCap ?? heartbeat.maxRunsPerDay, ), maxDailyCostCents: normalizeOptionalNonNegativeInteger( heartbeat.maxDailyCostCents ?? heartbeat.dailyCostCentsLimit ?? heartbeat.dailySpendCentsLimit ?? heartbeat.dailyBudgetCents, ), }; } function normalizeOptionalNonNegativeInteger(value: unknown) { if (value === null || value === undefined || value === "") return null; const normalized = Math.floor(asNumber(value, 0)); return normalized >= 0 ? normalized : null; } function currentUtcDayWindow(now = new Date()) { const start = new Date( Date.UTC( now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), 0, 0, 0, 0, ), ); const end = new Date( Date.UTC( now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1, 0, 0, 0, 0, ), ); return { start, end }; } async function getHeartbeatDailyCapBlock( agent: typeof agents.$inferSelect, policy: ReturnType, options: { checkRunCap?: boolean; checkCostCap?: boolean; excludeRunId?: string | null; } = {}, client: Pick = db, ) { const checkRunCap = options.checkRunCap ?? true; const checkCostCap = options.checkCostCap ?? true; const { start, end } = currentUtcDayWindow(); if (checkRunCap && policy.maxDailyRuns !== null) { const conditions = [ eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id), gte(heartbeatRuns.startedAt, start), lt(heartbeatRuns.startedAt, end), notInArray(heartbeatRuns.status, ["queued", "scheduled_retry"]), ]; if (options.excludeRunId) { conditions.push(sql`${heartbeatRuns.id} <> ${options.excludeRunId}`); } const [row] = await client .select({ total: sql`count(*)::integer` }) .from(heartbeatRuns) .where(and(...conditions)); const observed = Number(row?.total ?? 0); if (observed >= policy.maxDailyRuns) { return { reason: "heartbeat.daily_run_limit", observed, limit: policy.maxDailyRuns, }; } } if (checkCostCap && policy.maxDailyCostCents !== null) { const [row] = await client .select({ total: sql`coalesce(sum(${costEvents.costCents}), 0)::text`, }) .from(costEvents) .where( and( eq(costEvents.companyId, agent.companyId), eq(costEvents.agentId, agent.id), gte(costEvents.occurredAt, start), lt(costEvents.occurredAt, end), ), ); const observed = Number(row?.total ?? 0); if (compareCents(String(row?.total ?? 0), policy.maxDailyCostCents) >= 0) { return { reason: "heartbeat.daily_cost_limit", observed, limit: policy.maxDailyCostCents, }; } } return null; } async function cancelQueuedRunForHeartbeatDailyCap( run: typeof heartbeatRuns.$inferSelect, dailyCapBlock: NonNullable< Awaited> >, ) { const now = new Date(); const reason = "Cancelled because the agent reached a per-day heartbeat budget cap before adapter invocation"; const cancelled = await setRunStatus(run.id, "cancelled", { finishedAt: now, error: reason, errorCode: dailyCapBlock.reason, resultJson: { ...parseObject(run.resultJson), stopReason: dailyCapBlock.reason, observed: dailyCapBlock.observed, limit: dailyCapBlock.limit, effectiveTimeoutSec: 0, timeoutConfigured: false, timeoutSource: "heartbeat_daily_cap_gate", timeoutFired: false, }, }); if (!cancelled) return null; await setWakeupStatus(run.wakeupRequestId, "skipped", { finishedAt: now, error: reason, }); await appendRunEvent(cancelled, { eventType: "lifecycle", stream: "system", level: "warn", message: reason, payload: { reason: dailyCapBlock.reason, observed: dailyCapBlock.observed, limit: dailyCapBlock.limit, }, }); await releaseIssueExecutionAndPromote(cancelled, { suppressImmediateRecovery: true, }); return cancelled; } async function hasActionableTimerWork(agent: typeof agents.$inferSelect) { const row = await db .select({ id: issues.id }) .from(issues) .where( and( eq(issues.companyId, agent.companyId), eq(issues.assigneeAgentId, agent.id), isNull(issues.assigneeUserId), isNull(issues.hiddenAt), inArray(issues.status, [...TIMER_ACTIONABLE_ISSUE_STATUSES]), isNull(issues.conversationAgentId), nonIdleSlackIssueCondition(), ), ) .limit(1) .then((rows) => rows[0] ?? null); return Boolean(row); } async function markTimerHeartbeatChecked( agentId: string, source: WakeupOptions["source"], ) { if (source !== "timer") return; await db .update(agents) .set({ lastHeartbeatAt: new Date(), updatedAt: new Date(), }) .where(eq(agents.id, agentId)); } async function claimDueTimerHeartbeat( agent: typeof agents.$inferSelect, now: Date, intervalSec: number, ) { const dueBefore = new Date(now.getTime() - intervalSec * 1000); const claimed = await db .update(agents) .set({ lastHeartbeatAt: now, updatedAt: now, }) .where( and( eq(agents.id, agent.id), eq(agents.companyId, agent.companyId), or( lte(agents.lastHeartbeatAt, dueBefore), and( isNull(agents.lastHeartbeatAt), lte(agents.createdAt, dueBefore), ), ), ), ) .returning({ id: agents.id }) .then((rows) => rows[0] ?? null); if (!claimed) return null; return { wasFirstHeartbeat: !agent.lastHeartbeatAt }; } function issueRunPriorityRank(priority: string | null | undefined) { switch (priority) { case "critical": return 0; case "high": return 1; case "medium": return 2; case "low": return 3; default: return 4; } } async function listQueuedRunDependencyReadiness( companyId: string, queuedRuns: Array, ) { const issueIds = [ ...new Set( queuedRuns .map((run) => readNonEmptyString(parseObject(run.contextSnapshot).issueId), ) .filter((issueId): issueId is string => Boolean(issueId)), ), ]; if (issueIds.length === 0) { return new Map< string, Awaited> >(); } return issuesSvc.listDependencyReadiness(companyId, issueIds); } async function countRunningRunsForAgent(agentId: string) { const [{ count }] = await db .select({ count: sql`count(*)` }) .from(heartbeatRuns) .where( and( eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "running"), ), ); return Number(count ?? 0); } async function withChatControlRecoveryGate( run: typeof heartbeatRuns.$inferSelect, stage: "claim" | "dispatch", onClear: (tx: Db) => Promise, nativeRecovery = false, ) { const issueId = readNonEmptyString( parseObject(run.contextSnapshot).issueId, ); if (!issueId || run.invocationSource !== "automation") return onClear(db); await options.beforeChatControlRecoveryCheck?.({ runId: run.id, issueId, stage, }); if (stage === "dispatch" && nativeRecovery) { // Protected historical/already-admitted owners did not previously take // this fence's locks. A fresh exact read keeps unrelated Board contention // from turning their recovery into a new admission failure. const [current] = await db .select() .from(heartbeatRuns) .where( and( eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.agentId, run.agentId), eq(heartbeatRuns.status, "running"), ), ) .limit(1); if ( current && current.wakeupRequestId === run.wakeupRequestId && parseObject(current.contextSnapshot).issueId === issueId ) { const admission = readChatControlRecoveryAdmission(current); const expected = readChatControlRecoveryAdmission(run); if ( expected !== "invalid" && (admission === "admitted" || (admission === "historical" && expected === "historical")) ) return onClear(db); } } let terminal: typeof heartbeatRuns.$inferSelect | null = null; try { const attempt = () => db.transaction(async (tx) => { terminal = null; // Same queue-edit lock order, then the close committer's conversation // row. NOWAIT releases partial locks on contention. Claim defers to the // queue; dispatch retries this transaction before considering failure. const [issue] = await tx .select({ id: issues.id }) .from(issues) .where( and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)), ) .for("update", { noWait: true }) .limit(1); if (!issue) return null; if (run.wakeupRequestId) await tx .select({ id: agentWakeupRequests.id }) .from(agentWakeupRequests) .where( and( eq(agentWakeupRequests.id, run.wakeupRequestId), eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId), ), ) .for("update", { noWait: true }) .limit(1); const [current] = await tx .select() .from(heartbeatRuns) .where( and( eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.agentId, run.agentId), ), ) .for("update", { noWait: true }) .limit(1); if ( !current || current.status !== (stage === "claim" ? "queued" : "running") ) return null; const admission = readChatControlRecoveryAdmission(current); const expectedAdmission = readChatControlRecoveryAdmission(run); const admissionLost = expectedAdmission !== "historical" && admission === "historical"; if ( stage === "dispatch" && nativeRecovery && !admissionLost && (admission === "historical" || admission === "admitted") ) return onClear(tx as unknown as Db); // A new warm child can inherit a prior runner's PID during preparation. // Those fields do not prove this run's turn was dispatched. // Exact admitted/historical recovery retains its existing ownership; // a marked, unadmitted bootstrap must earn admission even after crash. const proof = admission === "invalid" || admissionLost ? { kind: "unresolved" as const } : await readChatControlRecoveryStop( tx as unknown as Db, { companyId: run.companyId, issueId, agentId: run.agentId, sourceRunId: run.id, }, true, ); if (proof.kind === "clear") { if (stage === "claim" || admission === "required") await tx .update(heartbeatRuns) .set({ runnerProfileJson: { ...parseObject(current.runnerProfileJson), [CHAT_CONTROL_RECOVERY_ADMISSION_KEY]: chatControlRecoveryAdmission( current, stage === "dispatch" ? "admitted" : admission === "admitted" ? "admitted" : "required", ), }, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, current.id)); return onClear(tx as unknown as Db); } if (proof.kind === "unresolved" && stage === "claim") return null; const now = new Date(); const stopped = proof.kind === "stopped"; const code = stopped ? CHAT_CONTROL_RECOVERY_STOP_CODE : CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE; const error = stopped ? "Automatic continuation stopped by the committed chat conversation close. Send a new request in chat or on the Board to start fresh work." : "Automatic continuation source could not be verified before provider admission. Review the task and send a fresh request; this attempt will not automatically retry."; [terminal] = await tx .update(heartbeatRuns) .set({ status: stopped ? "cancelled" : "failed", errorCode: code, error, finishedAt: now, resultJson: { ...parseObject(current.resultJson), automaticRecovery: { code, providerDispatched: false, ...(stopped ? { sourceRunId: proof.sourceRunId, conversationId: proof.conversationId, publicationId: proof.publicationId, } : {}), }, }, updatedAt: now, }) .where( and( eq(heartbeatRuns.id, current.id), eq(heartbeatRuns.status, current.status), ), ) .returning(); if (!terminal) return null; if (current.wakeupRequestId) await tx .update(agentWakeupRequests) .set({ status: stopped ? "skipped" : "failed", error, finishedAt: now, updatedAt: now, }) .where( and( eq(agentWakeupRequests.id, current.wakeupRequestId), eq(agentWakeupRequests.companyId, current.companyId), eq(agentWakeupRequests.agentId, current.agentId), eq(agentWakeupRequests.runId, current.id), ne(agentWakeupRequests.status, "cancelled"), ), ); await tx .update(issues) .set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, updatedAt: now, }) .where( and( eq(issues.companyId, current.companyId), eq(issues.id, issueId), eq(issues.executionRunId, current.id), ), ); return null; }); const result = stage === "dispatch" ? await retryChatControlAdmission(attempt) : await attempt(); if (terminal) { const settled = terminal as typeof heartbeatRuns.$inferSelect; publishLiveEvent({ companyId: settled.companyId, type: "heartbeat.run.status", payload: { runId: settled.id, agentId: settled.agentId, status: settled.status, errorCode: settled.errorCode, error: settled.error, }, }); publishRunLifecyclePluginEvent(settled); if (stage === "dispatch") await finalizeAgentStatus(settled.agentId, "cancelled"); } return result; } catch (error) { if (!isExternalChatWaitAuthorizationContention(error)) throw error; if (stage === "dispatch") { // No effect was admitted. Let the existing setup-failure path settle // this attempt distinctly; it must not become a successful close. throw new ChatControlRecoveryUnresolvedError(); } return null; } } async function claimQueuedRun( run: typeof heartbeatRuns.$inferSelect, companyAgents?: AgentOrgRow[], deferredPostCommitEffects?: WakeQueuePostCommitEffect[], ) { if (run.status !== "queued") return run; const [caseHandoffReceipt] = run.wakeupRequestId ? await db.select().from(caseHandoffs).where(and( eq(caseHandoffs.wakeupRequestId, run.wakeupRequestId), eq(caseHandoffs.companyId, run.companyId), )).limit(1) : []; if (caseHandoffReceipt) { const [current] = await db.select({ version: cases.handoffVersion, issueId: cases.workIssueId, agentId: issues.assigneeAgentId, statusVersion: issues.statusVersion }).from(cases) .innerJoin(issues, eq(issues.id, cases.workIssueId)).where(and(eq(cases.id, caseHandoffReceipt.caseId), eq(cases.companyId, run.companyId), eq(issues.companyId, run.companyId))).limit(1); if (!["pending_dispatch", "delivered"].includes(caseHandoffReceipt.state) || caseHandoffReceipt.toAgentId !== run.agentId || !current || current.version !== caseHandoffReceipt.version || current.issueId !== caseHandoffReceipt.issueId || current.agentId !== run.agentId || (!run.startedAt && current.statusVersion !== caseHandoffReceipt.appliedIssueStatusVersion)) { const [cancelled] = await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(), errorCode: "case_handoff_superseded", error: "This case handoff was cancelled or superseded.", updatedAt: new Date() }) .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.status, "queued"))).returning(); if (cancelled) await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: new Date(), error: cancelled.error }); return null; } if (!(await instanceSettings.getExperimental()).enableCases) return null; } const caseIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); if (caseIssueId) { const [pending] = await db.select().from(caseHandoffs).where(and(eq(caseHandoffs.companyId, run.companyId), eq(caseHandoffs.issueId, caseIssueId), inArray(caseHandoffs.state, ["awaiting_release", "pending_dispatch"]))).limit(1); if (pending && (pending.wakeupRequestId !== run.wakeupRequestId || pending.state === "awaiting_release")) return null; } // Claim is the first execution boundary. startedAt survives same-run // retries; restart recovery may also enter executeRun already running. // Neither may reinterpret an admitted, already-started message as new work. if (!run.startedAt && !(await slackMentionAllowsRunStart(db, run))) { const now = new Date(); const reason = "Message did not @mention the bot"; // This runs under the agent start lock. Do not use cancelRunInternal, // which recursively starts the next run and reacquires that lock. const cancelled = await setRunStatus(run.id, "cancelled", { finishedAt: now, error: reason, errorCode: "chat_mention_required", }); if (cancelled) { await setWakeupStatus(run.wakeupRequestId, "skipped", { finishedAt: now, error: reason }); await appendRunEvent(cancelled, { eventType: "lifecycle", stream: "system", level: "warn", message: reason, }); await releaseIssueExecutionAndPromote(cancelled, { suppressImmediateRecovery: true, deferredPostCommitEffects, }); } return null; } const agent = await getAgent(run.agentId); if (!agent) { await cancelRunInternal( run.id, "Cancelled because the agent no longer exists", ); return null; } // Keep accepted work in the durable queue until setup completes. if (isAgentAwaitingSetup(agent)) return null; const invokability = companyAgents ? evaluateAgentInvokability(toAgentOrgRow(agent), companyAgents) : await getAgentInvokability(agent); if (!invokability.invokable) { if (caseHandoffReceipt) return null; await cancelRunInternal( run.id, `Cancelled because the agent is not invokable: ${invokability.reason}`, ); return null; } if (!run.startedAt && !await canStartNewExternalBotWork(agent)) return null; const context = parseObject(run.contextSnapshot); const budgetBlock = await budgets.getInvocationBlock( run.companyId, run.agentId, { issueId: readNonEmptyString(context.issueId), projectId: readNonEmptyString(context.projectId), }, ); if (budgetBlock) { if (caseHandoffReceipt) return null; await cancelRunInternal(run.id, budgetBlock.reason); return null; } const dailyCapBlock = await getHeartbeatDailyCapBlock( agent, parseHeartbeatPolicy(agent), { excludeRunId: run.id, checkRunCap: true, checkCostCap: true, }, ); if (dailyCapBlock) { if (caseHandoffReceipt) return null; await cancelQueuedRunForHeartbeatDailyCap(run, dailyCapBlock); return null; } const issueId = readNonEmptyString(context.issueId); if (issueId && activeRunExecutions.size > 0) { // Native finalization publishes success before workspace synchronization, // provider suspension, and lease release finish. A queued comment must // not acquire a fresh sandbox while its predecessor still owns that work. // The executor's finally block retries this agent after removing its owner. const [settlingOwner] = await db.select({ id: heartbeatRuns.id }) .from(heartbeatRuns) .where(and( eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.agentId, run.agentId), eq(heartbeatRuns.runtimeMode, "native"), inArray(heartbeatRuns.id, [...activeRunExecutions]), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, )) .limit(1); if (settlingOwner) return null; } if (issueId) { const activePauseHold = await treeControlSvc.getActivePauseHoldGate( run.companyId, issueId, ); const treeHoldInteractionWake = activePauseHold && (await isVerifiedIssueTreeControlInteractionWake(db, { companyId: run.companyId, issueId, agentId: run.agentId, runId: run.id, wakeupRequestId: run.wakeupRequestId, contextSnapshot: context, })); if (activePauseHold && !treeHoldInteractionWake) { await cancelRunInternal( run.id, "Cancelled because issue is held by an active subtree pause hold", ); await logActivity(db, { companyId: run.companyId, actorType: "system", actorId: "system", agentId: run.agentId, runId: run.id, action: "issue.tree_hold_run_interrupted", entityType: "heartbeat_run", entityId: run.id, issueId: issueId, details: { issueId, holdId: activePauseHold.holdId, rootIssueId: activePauseHold.rootIssueId, source: "heartbeat.claim_queued_run", securityPrinciples: [ "Complete Mediation", "Fail Securely", "Secure Defaults", ], }, }); return null; } const dependencyReadiness = await issuesSvc.listDependencyReadiness( run.companyId, [issueId], ); const readiness = dependencyReadiness.get(issueId); const unresolvedBlockerCount = readiness?.unresolvedBlockerCount ?? 0; if ( unresolvedBlockerCount > 0 && !allowsIssueInteractionWake( context, ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS, ) ) { await cancelQueuedRunForBlockedDependencies( run, issueId, readiness?.unresolvedBlockerIssueIds ?? [], ); logger.info( { runId: run.id, issueId, unresolvedBlockerCount }, "claimQueuedRun: cancelled blocked queued run", ); return null; } const staleness = await runDispatch.cancelStaleQueuedRun({ runId: run.id, companyId: run.companyId, expectedStatus: "queued", }); if (staleness.outcome === "cancelled" || staleness.outcome === "deferred") { applyRunDispatchPostCommitEffects(staleness.postCommitEffects); logger.info( { runId: run.id, issueId, outcome: staleness.outcome, errorCode: staleness.outcome === "cancelled" ? staleness.errorCode : undefined, }, "claimQueuedRun: withheld queued run at the execution gate", ); return null; } } const claimedAt = new Date(); const responsibleUserId = await resolveResponsibleUserIdForRun({ run, contextSnapshot: context, issueContext: issueId ? await getIssueExecutionContext(run.companyId, issueId) : null, routineEnvContext: { routineId: null, env: null, responsibleUserId: null, }, }); // All ordinary and comment claims use the same company-scoped issue // lock. A batch may claim several runs before executeRun tracks any owner. async function lockIssueExecutionClaim(tx: Db) { const [owner] = issueId ? await tx.select({ assigneeAgentId: issues.assigneeAgentId, statusVersion: issues.statusVersion, executionRunId: issues.executionRunId, checkoutRunId: issues.checkoutRunId, }).from(issues).where(and( eq(issues.id, issueId), eq(issues.companyId, run.companyId), )).for("update") : []; const ownsIssue = owner?.assigneeAgentId === run.agentId && context.wakeReason !== "source_scoped_recovery_action"; const [handoff] = issueId ? await tx.select().from(caseHandoffs).where(and(eq(caseHandoffs.companyId, run.companyId), eq(caseHandoffs.issueId, issueId), inArray(caseHandoffs.state, ["awaiting_release", "pending_dispatch"]))).limit(1) : []; if (handoff && (handoff.wakeupRequestId !== run.wakeupRequestId || handoff.state === "awaiting_release")) return { ownsIssue, blocked: true }; if (caseHandoffReceipt) { const [current] = await tx.select({ state: caseHandoffs.state, version: caseHandoffs.version, currentVersion: cases.handoffVersion, workIssueId: cases.workIssueId, appliedIssueStatusVersion: caseHandoffs.appliedIssueStatusVersion, toAgentId: caseHandoffs.toAgentId, wakeupRequestId: caseHandoffs.wakeupRequestId }).from(caseHandoffs) .innerJoin(cases, eq(cases.id, caseHandoffs.caseId)).where(and(eq(caseHandoffs.id, caseHandoffReceipt.id), eq(caseHandoffs.companyId, run.companyId))).limit(1); if (!current || !["pending_dispatch", "delivered"].includes(current.state) || current.version !== current.currentVersion || current.workIssueId !== issueId || current.toAgentId !== run.agentId || current.wakeupRequestId !== run.wakeupRequestId || !ownsIssue || (!run.startedAt && current.appliedIssueStatusVersion !== owner?.statusVersion)) return { ownsIssue, blocked: true }; } if (ownsIssue && ["native_safe_replacement", "native_provider_overloaded"].includes(run.scheduledRetryReason ?? "") && owner.checkoutRunId && owner.checkoutRunId !== run.id) { return { ownsIssue, blocked: true }; } if (run.scheduledRetryReason === "native_provider_overloaded" && owner?.executionRunId && owner.executionRunId !== run.id && owner.executionRunId !== run.retryOfRunId) { return { ownsIssue, blocked: true }; } const previousRunId = run.scheduledRetryReason === "native_provider_overloaded" ? run.retryOfRunId : ownsIssue ? owner?.executionRunId : null; if (previousRunId && previousRunId !== run.id) { const [previous] = await tx.select({ status: heartbeatRuns.status }) .from(heartbeatRuns).where(and( eq(heartbeatRuns.id, previousRunId), eq(heartbeatRuns.companyId, run.companyId), )); // A terminal result can precede workspace/lease cleanup on this or // another controller. Local absence alone is not a release receipt. if (!isHeartbeatRunTerminalStatus(previous?.status) || liveRunExecutions.has(previousRunId)) { return { ownsIssue, blocked: true }; } if (await hasPendingRunCleanup(tx, run.companyId, previousRunId)) { return { ownsIssue, blocked: true }; } } return { ownsIssue, blocked: false }; } async function bindClaimedIssueExecution(tx: Db, ownsIssue: boolean, claimedRun: typeof heartbeatRuns.$inferSelect | null | undefined) { if (!claimedRun || !issueId || !ownsIssue) return; await tx.update(issues).set({ executionRunId: claimedRun.id, executionAgentNameKey: normalizeAgentNameKey(agent.name), executionLockedAt: claimedAt, updatedAt: claimedAt, }).where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))); } const nativeReviewContext = readNativeReviewAssignmentContext(context); const queuedCommentIds = queuedCommentIdsFromRunContext(context); if ( issueId && run.invocationSource === "automation" && queuedCommentIds.length > 0 ) await options.beforeChatControlRecoveryCheck?.({ runId: run.id, issueId, stage: "claim", }); const queuedCommentClaim = !nativeReviewContext && issueId && run.wakeupRequestId && queuedCommentIds.length > 0 ? await db .transaction(async (tx) => { // Match the queue-edit lock order: issue, wake, then run. Once the // run becomes running, a concurrent discard must observe the // claimed wake and return an explicit conflict; if discard wins, // this claim observes the cancelled queue and does no work. const issueClaim = await lockIssueExecutionClaim(tx as unknown as Db); if (issueClaim.blocked) return { kind: "stale" as const, run: null }; const wake = await tx .select() .from(agentWakeupRequests) .where( and( eq(agentWakeupRequests.id, run.wakeupRequestId!), eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId), ), ) .for("update") .limit(1) .then((rows) => rows[0] ?? null); const lockedRun = await tx .select() .from(heartbeatRuns) .where( and( eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.agentId, run.agentId), ), ) .for("update") .limit(1) .then((rows) => rows[0] ?? null); if ( !wake || wake.status !== "queued" || wake.runId !== run.id || !lockedRun || lockedRun.status !== "queued" || lockedRun.wakeupRequestId !== wake.id ) { return { kind: "stale" as const, run: null }; } if (lockedRun.invocationSource === "automation") { const admission = readChatControlRecoveryAdmission(lockedRun); if (admission === "invalid") return { kind: "stale" as const, run: null }; const proof = await readChatControlRecoveryStop( tx as unknown as Db, { companyId: lockedRun.companyId, issueId, agentId: lockedRun.agentId, sourceRunId: lockedRun.id, }, true, ); if (proof.kind === "unresolved") return { kind: "stale" as const, run: null }; if (proof.kind === "stopped") { const error = "Automatic continuation stopped by the committed chat conversation close. Send a fresh request in chat or on the Board."; await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); const [cancelled] = await tx .update(heartbeatRuns) .set({ status: "cancelled", errorCode: CHAT_CONTROL_RECOVERY_STOP_CODE, error, finishedAt: claimedAt, resultJson: { ...parseObject(lockedRun.resultJson), automaticRecovery: { code: CHAT_CONTROL_RECOVERY_STOP_CODE, providerDispatched: false, sourceRunId: proof.sourceRunId, conversationId: proof.conversationId, publicationId: proof.publicationId, }, }, updatedAt: claimedAt, }) .where( and( eq(heartbeatRuns.id, lockedRun.id), eq(heartbeatRuns.status, "queued"), ), ) .returning(); if (!cancelled) return { kind: "stale" as const, run: null }; await tx .update(agentWakeupRequests) .set({ status: "skipped", error, finishedAt: claimedAt, updatedAt: claimedAt, }) .where(eq(agentWakeupRequests.id, wake.id)); await tx .update(issues) .set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, updatedAt: claimedAt, }) .where( and( eq(issues.id, issueId), eq(issues.companyId, run.companyId), eq(issues.executionRunId, run.id), ), ); return { kind: "cancelled" as const, run: cancelled }; } await tx .update(heartbeatRuns) .set({ runnerProfileJson: { ...parseObject(lockedRun.runnerProfileJson), [CHAT_CONTROL_RECOVERY_ADMISSION_KEY]: chatControlRecoveryAdmission( lockedRun, admission === "admitted" ? "admitted" : "required", ), }, updatedAt: claimedAt, }) .where(eq(heartbeatRuns.id, lockedRun.id)); } const authoritativeIds = queuedCommentIdsFromWakePayload( wake.payload, ); if (authoritativeIds.length === 0) { // Legacy/direct comment wakes carry comment ids in their ordinary // payload and context, not in the authoritative queued-message // envelope. Preserve their established claim path; only an // explicitly bound queued-message envelope is subject to the // live-comment discard gate below. await withExternalAdmissionGuard(tx, run.companyId, run.agentId, () => assertNoExternalOverlap(tx, run.companyId, run.agentId)); await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); const [claimedRun] = await tx .update(heartbeatRuns) .set({ status: "running", runnerProfileJson: sql`(case when jsonb_typeof(${heartbeatRuns.runnerProfileJson}) = 'object' then ${heartbeatRuns.runnerProfileJson} else '{}'::jsonb end) || ${JSON.stringify({ adapterDispatch: { adapterType: agent.adapterType } })}::jsonb`, ...legacyControllerClaim(run.runtimeMode), responsibleUserId, startedAt: lockedRun.startedAt ?? claimedAt, updatedAt: claimedAt, }) .where( and( eq(heartbeatRuns.id, lockedRun.id), eq(heartbeatRuns.status, "queued"), ), ) .returning(); await bindClaimedIssueExecution(tx as unknown as Db, issueClaim.ownsIssue, claimedRun); return claimedRun ? { kind: "claimed" as const, run: claimedRun } : { kind: "stale" as const, run: null }; } const commentRows = await tx .select({ id: issueComments.id, deletedAt: issueComments.deletedAt, }) .from(issueComments) .where( and( eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId), inArray(issueComments.id, authoritativeIds), ), ); const liveIds = authoritativeIds.filter((commentId) => { const comment = commentRows.find((row) => row.id === commentId); return Boolean(comment && !comment.deletedAt); }); if (liveIds.length === 0) { const reason = "Queued messages were discarded before dispatch"; await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); const [cancelled] = await tx .update(heartbeatRuns) .set({ status: "cancelled", finishedAt: claimedAt, error: reason, errorCode: "queued_comment_discarded", updatedAt: claimedAt, }) .where( and( eq(heartbeatRuns.id, lockedRun.id), eq(heartbeatRuns.status, "queued"), ), ) .returning(); await tx .update(agentWakeupRequests) .set({ status: "cancelled", finishedAt: claimedAt, error: reason, updatedAt: claimedAt, }) .where(eq(agentWakeupRequests.id, wake.id)); await tx .update(issues) .set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, updatedAt: claimedAt, }) .where( and( eq(issues.id, issueId), eq(issues.companyId, run.companyId), eq(issues.executionRunId, run.id), ), ); return { kind: "cancelled" as const, run: cancelled ?? lockedRun, }; } await withExternalAdmissionGuard(tx, run.companyId, run.agentId, () => assertNoExternalOverlap(tx, run.companyId, run.agentId)); await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); await tx .update(agentWakeupRequests) .set({ status: "claimed", claimedAt, payload: withQueuedCommentIdsInWakePayload( wake.payload, liveIds, ), updatedAt: claimedAt, }) .where(eq(agentWakeupRequests.id, wake.id)); const [claimedRun] = await tx .update(heartbeatRuns) .set({ status: "running", runnerProfileJson: sql`(case when jsonb_typeof(${heartbeatRuns.runnerProfileJson}) = 'object' then ${heartbeatRuns.runnerProfileJson} else '{}'::jsonb end) || ${JSON.stringify({ adapterDispatch: { adapterType: agent.adapterType } })}::jsonb`, ...legacyControllerClaim(run.runtimeMode), responsibleUserId, startedAt: lockedRun.startedAt ?? claimedAt, contextSnapshot: withQueuedCommentIdsInRunContext( lockedRun.contextSnapshot, liveIds, ), updatedAt: claimedAt, }) .where( and( eq(heartbeatRuns.id, lockedRun.id), eq(heartbeatRuns.status, "queued"), ), ) .returning(); await bindClaimedIssueExecution(tx as unknown as Db, issueClaim.ownsIssue, claimedRun); return claimedRun ? { kind: "claimed" as const, run: claimedRun } : { kind: "stale" as const, run: null }; }) .catch((error) => { if (isExternalChatWaitAuthorizationContention(error)) return { kind: "stale" as const, run: null }; throw error; }) : null; if (queuedCommentClaim?.kind === "cancelled") { await appendRunEvent(queuedCommentClaim.run, { eventType: "lifecycle", stream: "system", level: "warn", message: queuedCommentClaim.run.error ?? "Queued messages were discarded before dispatch", }); publishLiveEvent({ companyId: queuedCommentClaim.run.companyId, type: "heartbeat.run.status", payload: { runId: queuedCommentClaim.run.id, agentId: queuedCommentClaim.run.agentId, status: queuedCommentClaim.run.status, invocationSource: queuedCommentClaim.run.invocationSource, triggerDetail: queuedCommentClaim.run.triggerDetail, error: queuedCommentClaim.run.error ?? null, errorCode: queuedCommentClaim.run.errorCode ?? null, startedAt: queuedCommentClaim.run.startedAt ? new Date(queuedCommentClaim.run.startedAt).toISOString() : null, finishedAt: queuedCommentClaim.run.finishedAt ? new Date(queuedCommentClaim.run.finishedAt).toISOString() : null, }, }); publishRunLifecyclePluginEvent(queuedCommentClaim.run); // Fire-and-forget: nothing else in this path depends on the emission, // so it must not delay the return. void emitAgentTaskRun(db, queuedCommentClaim.run); return null; } const claimed = queuedCommentClaim ? queuedCommentClaim.run : await withChatControlRecoveryGate(run, "claim", async (tx) => { const claimValues = { status: "running", runnerProfileJson: sql`(case when jsonb_typeof(${heartbeatRuns.runnerProfileJson}) = 'object' then ${heartbeatRuns.runnerProfileJson} else '{}'::jsonb end) || ${JSON.stringify({ adapterDispatch: { adapterType: agent.adapterType } })}::jsonb`, ...legacyControllerClaim(run.runtimeMode), responsibleUserId, startedAt: run.startedAt ?? claimedAt, updatedAt: claimedAt, }; if (nativeReviewContext) { if (run.scheduledRetryReason === "native_provider_overloaded") { const predecessor = await lockIssueExecutionClaim(tx); if (predecessor.blocked) return null; } return claimQueuedNativeReviewRun(tx, { run, claimedAt, claimValues, agentNameKey: normalizeAgentNameKey(agent.name), }); } return tx.transaction(async (claimTx) => { const issueClaim = await lockIssueExecutionClaim(claimTx as unknown as Db); if (issueClaim.blocked) return null; if (run.wakeupRequestId) await claimTx.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( eq(agentWakeupRequests.id, run.wakeupRequestId), eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId), )).for("update"); const [lockedRun] = await claimTx.select().from(heartbeatRuns).where(and( eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.agentId, run.agentId), )).for("update"); if (!lockedRun || lockedRun.status !== "queued") return null; await withExternalAdmissionGuard(claimTx, run.companyId, run.agentId, () => assertNoExternalOverlap(claimTx, run.companyId, run.agentId)); await notifyChatRunMilestones(claimTx, { companyId: run.companyId, issueId: issueId ?? undefined }); const claimedRun = await claimTx.update(heartbeatRuns).set(claimValues).where(and( eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "queued"), )).returning().then((rows) => rows[0] ?? null); await bindClaimedIssueExecution(claimTx as unknown as Db, issueClaim.ownsIssue, claimedRun); return claimedRun; }); }); if (!claimed) return null; publishLiveEvent({ companyId: claimed.companyId, type: "heartbeat.run.status", payload: { runId: claimed.id, agentId: claimed.agentId, status: claimed.status, invocationSource: claimed.invocationSource, triggerDetail: claimed.triggerDetail, error: claimed.error ?? null, errorCode: claimed.errorCode ?? null, startedAt: claimed.startedAt ? new Date(claimed.startedAt).toISOString() : null, finishedAt: claimed.finishedAt ? new Date(claimed.finishedAt).toISOString() : null, }, }); publishRunLifecyclePluginEvent(claimed); if (!nativeReviewContext) { await setWakeupStatus(claimed.wakeupRequestId, "claimed", { claimedAt }); } // Fix A (lazy locking): stamp executionRunId now that the run is actually running, // not at queue time. Guard is idempotent — safe if called more than once. const claimedContext = parseObject(claimed.contextSnapshot); const claimedIssueId = readNonEmptyString(claimedContext.issueId); const claimedWakeReason = readNonEmptyString(claimedContext.wakeReason); if ( !nativeReviewContext && claimedIssueId && claimedWakeReason !== "source_scoped_recovery_action" ) { const claimedAgent = await getAgent(claimed.agentId); await db .update(issues) .set({ executionRunId: claimed.id, executionAgentNameKey: normalizeAgentNameKey(claimedAgent?.name), executionLockedAt: claimedAt, updatedAt: claimedAt, }) .where( and( eq(issues.id, claimedIssueId), eq(issues.companyId, claimed.companyId), // Mention/context runs can touch an issue, but only the current assignee // owns the issue execution lock shown as the active run. eq(issues.assigneeAgentId, claimed.agentId), ["native_safe_replacement", "native_provider_overloaded"].includes(claimed.scheduledRetryReason ?? "") ? or( isNull(issues.checkoutRunId), eq(issues.checkoutRunId, claimed.id), ) : undefined, or( isNull(issues.executionRunId), eq(issues.executionRunId, claimed.id), ), ), ); } return claimed; } // startNextQueuedRunForAgent checks admission suppression once, then claims // runs (sets status "running"), then dispatches each to executeRun, which // checks suppression again before it does any work. Suppression (task // drain, worktree mode, a database restore) can start in the gap between // those two checks. When executeRun's check catches that, the run is // already claimed — release it back to "queued" so it does not keep a // running execution lock that nothing will ever process. This runs inside // the same promise startNextQueuedRunForAgent already tracks in // activeRunExecutionPromises, so getTaskDrainStatus() keeps reporting the // run as active until the release finishes. // // The run row, the wakeup request, and the issue execution lock all guard // the same claim, so one transaction commits all three writes together. A // partial write (for example the run flips to "queued" but the wakeup or // issue update then fails) would let the execution promise clear from // activeRunExecutionPromises while the wakeup stayed "claimed" or the // issue stayed locked to a queued run — task-drain status would then read // quiescent while the database still held part of the old claim. async function releaseRunClaimedJustBeforeSuppression(runId: string) { const now = new Date(); await db.transaction(async (tx) => { const [candidate] = await tx.select({ companyId: heartbeatRuns.companyId, contextSnapshot: heartbeatRuns.contextSnapshot }) .from(heartbeatRuns).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.status, "running"))).for("update"); if (!candidate) return; await notifyChatRunMilestones(tx, { companyId: candidate.companyId, issueId: readNonEmptyString(parseObject(candidate.contextSnapshot).issueId) ?? undefined }); const released = await tx .update(heartbeatRuns) .set({ status: "queued", startedAt: null, responsibleUserId: null, updatedAt: now, }) .where( and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.status, "running")), ) .returning() .then((rows) => rows[0] ?? null); if (!released) return; if (released.wakeupRequestId) { await tx .update(agentWakeupRequests) .set({ status: "queued", claimedAt: null, updatedAt: now }) .where(and(eq(agentWakeupRequests.id, released.wakeupRequestId), ne(agentWakeupRequests.status, "cancelled"))); } const context = parseObject(released.contextSnapshot); const issueId = readNonEmptyString(context.issueId); if (issueId) { await tx .update(issues) .set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, updatedAt: now, }) .where( and( eq(issues.id, issueId), eq(issues.companyId, released.companyId), eq(issues.executionRunId, released.id), ), ); } }); } async function cancelQueuedRunForBlockedDependencies( run: typeof heartbeatRuns.$inferSelect, issueId: string, unresolvedBlockerIssueIds: string[], ) { const now = new Date(); const reason = "Cancelled because issue dependencies are still blocked; Paperclip will wake the assignee when blockers resolve"; const cancelled = await setRunStatus(run.id, "cancelled", { finishedAt: now, error: reason, errorCode: "issue_dependencies_blocked", resultJson: { ...parseObject(run.resultJson), stopReason: "issue_dependencies_blocked", effectiveTimeoutSec: 0, timeoutConfigured: false, timeoutSource: "dependency_gate", timeoutFired: false, }, }); if (!cancelled) return null; await setWakeupStatus(run.wakeupRequestId, "skipped", { finishedAt: now, error: reason, }); await db .update(issues) .set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, updatedAt: now, }) .where( and( eq(issues.companyId, run.companyId), eq(issues.id, issueId), eq(issues.executionRunId, run.id), ), ); await appendRunEvent(cancelled, { eventType: "lifecycle", stream: "system", level: "warn", message: reason, payload: { issueId, unresolvedBlockerIssueIds, }, }); return cancelled; } async function resumeQueuedRuns() { if ((await getSchedulingSuppression()).suppressed) return; await resumeExecutionWaitComments(); const cutoff = await getWorktreeExecutionCutoff(); const pendingInterrupts = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId }) .from(agentWakeupRequests).innerJoin(companies, eq(companies.id, agentWakeupRequests.companyId)) .where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"), eq(companies.status, "active"), sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`, lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)), cutoff ? gte(agentWakeupRequests.requestedAt, cutoff) : undefined)) .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); for (const wake of pendingInterrupts) { await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(and( eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution"), )); await resumeQueuedCommentInterrupt(wake.companyId, wake.id).catch(err => { logger.warn({ err, queueId: wake.id }, "failed to resume interrupted comment queue"); }); } // A server restart or a message/cleanup race can leave a deferred wake // after its owner has released the issue lock. Revisit it through the same // release admission, so recovery holds and operator Stops still apply. const strandedQueues = await db.select({ wake: agentWakeupRequests }) .from(agentWakeupRequests) .innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId), sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`, or(eq(issues.assigneeAgentId, agentWakeupRequests.agentId), sql`${agentWakeupRequests.payload} #>> '{_paperclipWakeContext,wakeReason}' = 'issue_comment_mentioned'`))) .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active"))) .where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"), isNull(issues.executionRunId), or(and( sql`jsonb_typeof(${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}') = 'array'`, sql`${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`, ), sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`), sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is null`, sql`NOT EXISTS (SELECT 1 FROM heartbeat_runs live_issue_run WHERE live_issue_run.company_id = ${issues.companyId} AND (live_issue_run.context_snapshot->>'issueId' = ${issues.id}::text OR live_issue_run.native_issue_id = ${issues.id}) AND live_issue_run.status IN ('queued', 'running', 'scheduled_retry'))`, cutoff ? gte(agentWakeupRequests.requestedAt, cutoff) : undefined)) .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); for (const { wake } of strandedQueues) { if (!queuedCommentIdsFromWakePayload(wake.payload).length && !await readQueuedInteractionResponse(db, wake.companyId, String(wake.payload?.issueId), wake.payload)) continue; const [latest] = await db.select().from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, wake.companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${String(wake.payload?.issueId)}`, or(isNull(heartbeatRuns.nativeIssueId), sql`${heartbeatRuns.nativeIssueId}::text = ${String(wake.payload?.issueId)}`), )).orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(and( eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution"), )); if (!latest || !latest.finishedAt || !isHeartbeatRunTerminalStatus(latest.status)) continue; if (latest.runtimeMode !== "legacy" && latest.runtimeMode !== "native") continue; if (latest.runtimeMode === "legacy") { const cancelledAdmission = latest.status === "cancelled" && !latest.startedAt && latest.errorCode === "execution_reconciliation_required"; if ((latest.status !== "cancelled" || cancelledAdmission) && await getExecutionBlocker(db, wake.companyId, String(wake.payload?.issueId))) { await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => { logger.warn({ err, queueId: wake.id }, "failed to deliver saved legacy comment after recovery stopped"); }); } } await releaseIssueExecutionAndPromote(latest, { suppressImmediateRecovery: true, resumeStrandedQueue: true }).catch(err => { logger.warn({ err, queueId: wake.id }, "failed to promote stranded comments"); }); } // The cancellation marker is durable intent. Retry while its exact queue // is still deferred, including after a failed cleanup promotion or restart. // Normal admission still checks process ownership, leases, pauses, and scope. const interruptedQueues = await db .select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId }) .from(agentWakeupRequests) .innerJoin(heartbeatRuns, and( sql`${heartbeatRuns.resultJson}->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text`, eq(heartbeatRuns.companyId, agentWakeupRequests.companyId), eq(heartbeatRuns.agentId, agentWakeupRequests.agentId), )) .innerJoin(companies, eq(companies.id, heartbeatRuns.companyId)) .where(and( eq(agentWakeupRequests.status, "deferred_issue_execution"), eq(heartbeatRuns.status, "cancelled"), eq(heartbeatRuns.runtimeMode, "legacy"), eq(companies.status, "active"), cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined, )); for (const run of interruptedQueues) { await releaseIssueExecutionAndPromote(run, { suppressImmediateRecovery: true }).catch((err) => { logger.error({ err, runId: run.id }, "failed to retry interrupted comment queue"); }); } const queuedRuns = await db .select({ agentId: heartbeatRuns.agentId }) .from(heartbeatRuns) .innerJoin(companies, eq(companies.id, heartbeatRuns.companyId)) .where( and( eq(heartbeatRuns.status, "queued"), eq(companies.status, "active"), cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined, ), ); const agentIds = [...new Set(queuedRuns.map((r) => r.agentId))]; for (const agentId of agentIds) { await startNextQueuedRunForAgent(agentId); } } // A 403 from claimQueuedRun comes from the run's own persisted identity // (an unverifiable interrupt receipt, a manual wake with no user). Those rows // do not change, so the claim fails the same way on every pass and restart. function isPermanentClaimRejection(err: unknown): err is HttpError { return err instanceof HttpError && err.status === 403; } // Other 4xx rejections can clear later (a responsible user gets assigned, a // conflicting claim finishes). Keep the run queued, but do not let it stop // the rest of the queue or startup recovery. function isDeferrableClaimRejection(err: unknown): err is HttpError { return err instanceof HttpError && err.status >= 400 && err.status < 500; } // Settle runs that can never be claimed. Letting the error escape stalls the // agent's queue and, during startup recovery, stops the server from booting. async function cancelRejectedQueuedRuns( rejected: Array<{ run: typeof heartbeatRuns.$inferSelect; err: HttpError }>, ) { for (const { run, err } of rejected) { logger.warn( { err, runId: run.id, agentId: run.agentId, companyId: run.companyId }, "cancelling queued heartbeat run whose claim was rejected", ); try { await cancelRunInternal( run.id, `Cancelled because the queued run cannot be claimed: ${err.message}`, { errorCode: "queued_run_claim_rejected" }, ); } catch (cancelErr) { logger.error( { err: cancelErr, runId: run.id }, "failed to cancel queued heartbeat run whose claim was rejected; it stays queued for the next recovery pass", ); } } } async function startNextQueuedRunForAgent(agentId: string) { if ((await getSchedulingSuppression()).suppressed) return []; const cutoff = await getWorktreeExecutionCutoff(); // Cancelled after the start lock is released: cancelRunInternal promotes the // agent's next queued run, which takes this same lock. const rejectedClaims: Array<{ run: typeof heartbeatRuns.$inferSelect; err: HttpError }> = []; const deferredPostCommitEffects: WakeQueuePostCommitEffect[] = []; let cancellationReason: string | undefined; return withAgentStartLock(agentId, async () => { const agent = await getAgent(agentId); if (!agent) return []; const invokability = await getAgentInvokability(agent); if (!invokability.invokable) { if (shouldCancelRunsForNonInvokableAgent(invokability)) { cancellationReason = `Cancelled because the agent is not invokable: ${invokability.reason}`; } return []; } const policy = parseHeartbeatPolicy(agent); const runningCount = await countRunningRunsForAgent(agentId); const availableSlots = Math.max( 0, policy.maxConcurrentRuns - runningCount, ); if (availableSlots <= 0) return []; const queuedRuns = await db .select() .from(heartbeatRuns) .where( and( eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "queued"), cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined, ), ) .orderBy(asc(heartbeatRuns.createdAt)); if (queuedRuns.length === 0) return []; if (queuedRuns.every(run => !run.startedAt) && !await canStartNewExternalBotWork(agent)) return []; const dependencyReadiness = await listQueuedRunDependencyReadiness( agent.companyId, queuedRuns, ); const queuedIssueIds = [ ...new Set( queuedRuns .map((run) => readNonEmptyString(parseObject(run.contextSnapshot).issueId), ) .filter((issueId): issueId is string => Boolean(issueId)), ), ]; const issueRows = await db .select({ id: issues.id, status: issues.status, priority: issues.priority, }) .from(issues) .where( queuedIssueIds.length > 0 ? and( eq(issues.companyId, agent.companyId), inArray(issues.id, queuedIssueIds), ) : sql`false`, ); const issueById = new Map(issueRows.map((row) => [row.id, row])); const companyAgents = await listCompanyAgentOrgRows(agent.companyId); const prioritizedRuns = [...queuedRuns].sort((left, right) => { const leftIssueId = readNonEmptyString( parseObject(left.contextSnapshot).issueId, ); const rightIssueId = readNonEmptyString( parseObject(right.contextSnapshot).issueId, ); const leftReadiness = leftIssueId ? dependencyReadiness.get(leftIssueId) : null; const rightReadiness = rightIssueId ? dependencyReadiness.get(rightIssueId) : null; const leftReady = leftIssueId ? (leftReadiness?.isDependencyReady ?? true) : true; const rightReady = rightIssueId ? (rightReadiness?.isDependencyReady ?? true) : true; const leftIssue = leftIssueId ? issueById.get(leftIssueId) : null; const rightIssue = rightIssueId ? issueById.get(rightIssueId) : null; const leftRank = leftIssueId ? leftReady ? leftIssue?.status === "in_progress" ? 0 : 1 : 3 : 2; const rightRank = rightIssueId ? rightReady ? rightIssue?.status === "in_progress" ? 0 : 1 : 3 : 2; if (leftRank !== rightRank) return leftRank - rightRank; const leftPriorityRank = issueRunPriorityRank(leftIssue?.priority); const rightPriorityRank = issueRunPriorityRank(rightIssue?.priority); if (leftPriorityRank !== rightPriorityRank) return leftPriorityRank - rightPriorityRank; return left.createdAt.getTime() - right.createdAt.getTime(); }); const claimedRuns: Array = []; for (const queuedRun of prioritizedRuns) { if (claimedRuns.length >= availableSlots) break; let claimed: typeof heartbeatRuns.$inferSelect | null; try { claimed = await claimQueuedRun(queuedRun, companyAgents, deferredPostCommitEffects); } catch (err) { if (isPermanentClaimRejection(err)) { rejectedClaims.push({ run: queuedRun, err }); continue; } if (!isDeferrableClaimRejection(err)) throw err; logger.warn( { err, runId: queuedRun.id, agentId: queuedRun.agentId, companyId: queuedRun.companyId }, "queued heartbeat run claim was rejected; leaving it queued for the next recovery pass", ); continue; } if (claimed) claimedRuns.push(claimed); } if (claimedRuns.length === 0) return []; for (const claimedRun of claimedRuns) { const execution = executeRun(claimedRun.id).catch((err) => { logger.error( { err, runId: claimedRun.id }, "queued heartbeat execution failed", ); }); // Register the in-flight execution so drainActiveRunExecutions() can await // it. executeRun resolves only after its finally block finishes flushing // run rows/events, so awaiting this promise guarantees the run's writes // have landed before a caller (e.g. a test's afterEach) mutates the DB. activeRunExecutionPromises.add(execution); void execution.finally(() => { // drainActiveRunExecutions loops on activeRunExecutionPromises.size, // so an entry that never clears here would hang it forever. activeRunExecutionPromises.delete(execution); }); } return claimedRuns; }).finally(async () => { // Dispatch promoted inputs after the agent start lock is released. try { await applyWakeQueuePostCommitEffects(deferredPostCommitEffects); } finally { if (cancellationReason) await cancelActiveForAgentInternal(agentId, cancellationReason); await cancelRejectedQueuedRuns(rejectedClaims); } }); } // Public wakeup entry point. Callers dispatch it fire-and-forget, so register // the promise in activeWakeupPromises before it starts its asynchronous // prologue. drainActiveRunExecutions can then await a wake that is still before // run registration. Internal callers reference enqueueWakeup directly and // already await it, so they do not need this registration. function trackWakeup( agentId: string, opts: WakeupOptions = {}, ): ReturnType { const promise = enqueueWakeup(agentId, opts); activeWakeupPromises.add(promise); void promise .catch(() => {}) .finally(() => { activeWakeupPromises.delete(promise); }); return promise; } async function enqueueWakeup(agentId: string, opts: WakeupOptions = {}, executionWaitRequestId?: string) { const source = opts.source ?? "on_demand"; const triggerDetail = opts.triggerDetail ?? null; const contextSnapshot: Record = { ...(opts.contextSnapshot ?? {}), }; const reason = opts.reason ?? null; let payload = opts.payload ? { ...opts.payload } : null; // Only the board queue route can record interruption authority on an // existing receipt. Never accept this internal marker from a wake caller. if (payload) { delete payload.queuedCommentInterrupt; delete payload.manualUserWake; delete payload.taskRetry; } if (opts.manualUserWake) { if (opts.requestedByActorType !== "user" || !opts.requestedByActorId || opts.failedRunId) { throw new HttpError(403, "Manual wake requires an authenticated user"); } payload = { ...payload, manualUserWake: true }; } const executionReconciliationWake = contextSnapshot.source === "execution.reconciled" || opts.idempotencyKey?.startsWith("execution-reconciliation:") === true; const { contextSnapshot: enrichedContextSnapshot, issueIdFromPayload, taskKey, wakeCommentId, } = enrichWakeContextSnapshot({ contextSnapshot, reason, source, triggerDetail, payload, }); // Keep each request's own server-derived origin, including coalesced wakes. // A run's merged context cannot establish which caller authored one receipt. // Overwrite caller-supplied nested context rather than trusting it. payload = { ...payload, [DEFERRED_WAKE_CONTEXT_KEY]: { ...enrichedContextSnapshot } }; let issueId = readNonEmptyString(enrichedContextSnapshot.issueId) ?? issueIdFromPayload; if (executionReconciliationWake && !issueId) return null; const durableRequest = opts.durableChatRequest; let taskRetryActorId: string | null = null; if (opts.taskRetryRequestId) { const [receipt] = await db.select().from(agentWakeupRequests).where(and( eq(agentWakeupRequests.id, opts.taskRetryRequestId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), eq(agentWakeupRequests.requestedByActorType, "user"), sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId ?? ""}`, )); const retry = readTaskRetryIntent(receipt?.payload); if (!receipt || retry?.generationId !== receipt.id || retry.canonicalIntentId || retry.actorId !== receipt.requestedByActorId) return null; const chatRetry = retry.chatActionId && durableRequest?.id === retry.chatActionId && durableRequest.failedRunRetry?.failedRunId === retry.sourceRunId; if (retry.chatActionId ? !chatRetry : opts.requestedByActorType !== "user" || retry.actorId !== opts.requestedByActorId) return null; taskRetryActorId = retry.actorId; enrichedContextSnapshot.forceFreshSession = true; enrichedContextSnapshot.previousRunId = retry.sourceRunId; enrichedContextSnapshot.taskRetryRequestId = receipt.id; } let agent = await getAgent(agentId); if (!agent) throw notFound("Agent not found"); if (issueId) { const [pending] = await db.select().from(caseHandoffs).where(and(eq(caseHandoffs.companyId, agent.companyId), eq(caseHandoffs.issueId, issueId), inArray(caseHandoffs.state, ["awaiting_release", "pending_dispatch"]))).limit(1); if (pending && opts.durableCaseHandoffRequest?.handoffId !== pending.id) return null; } // Mentions only annotate comments. Ignore legacy callers before creating // a run or deferred request; assignment and review have their own wakes. if (reason === "issue_comment_mentioned" || enrichedContextSnapshot.wakeReason === "issue_comment_mentioned") return null; if (issueId) { const conversation = await getIssueExecutionContext(agent.companyId, issueId); if (taskRetryActorId && conversation) await assertTaskRetryConversationAllowed(db, conversation, taskRetryActorId); if (reason === "issue_children_completed" && conversation?.originKind === "onboarding_first_task") enrichedContextSnapshot.onboardingCompletion = true; if (isConversation(conversation)) { if (opts.manualUserWake && conversation!.conversationUserId !== opts.requestedByActorId) { throw new HttpError(403, "Only the conversation owner can start a chat run"); } if (isConversationExecutionWake(conversation, reason ?? readNonEmptyString(enrichedContextSnapshot.wakeReason))) return null; if (agent.id !== conversation!.conversationAgentId) return null; if (!(await instanceSettings.getExperimental()).enableAgentChat) return null; if (!wakeCommentId && !opts.taskRetryRequestId && isWaitingConversation(conversation) && !hasInteractionContinuationWakeContext(enrichedContextSnapshot) && reason !== CHAT_COMPLETION_WAKE_REASON) return null; } } if (agent.adapterType === "paperclip_runner") { const oldConfig = parseObject(agent.adapterConfig); const nextConfig = normalizeLegacyRunnerProvider(oldConfig); if (nextConfig !== oldConfig) { await agentService(db).update( agent.id, { adapterConfig: nextConfig }, { recordRevision: { source: "normalize_runner_provider", createdByAgentId: null, createdByUserId: null, }, }, ); await logActivity(db, { companyId: agent.companyId, actorType: "system", actorId: "heartbeat", action: "agent.updated", entityType: "agent", entityId: agent.id, details: { provider: "codex", reason: "native_codex_provider" }, }); agent = (await getAgent(agentId))!; } } if (opts.failedRunId) { const failed = await getRun(opts.failedRunId, { includeExecutionEvidence: true }); if (opts.requestedByActorType !== "user" || !opts.requestedByActorId || reason !== "retry_failed_run" || source !== "on_demand" || triggerDetail !== "manual" || !failed || failed.companyId !== agent.companyId || failed.agentId !== agentId || !(await canRetryStoppedRun(db, failed)) || (failed.nativeIssueId ?? readNonEmptyString(failed.contextSnapshot?.issueId)) !== issueId) { throw conflict("The selected failed run cannot be retried for this task."); } if (!activeRunExecutions.has(failed.id) && !adapterExecutionControls.has(failed.id)) { await sweepPendingCleanupLeases({ explicitRetry: { companyId: failed.companyId, runId: failed.id, actorId: opts.requestedByActorId, } }); } if (isConversationAdapter(agent.adapterType) || agent.adapterType === "paperclip_runner") { enrichedContextSnapshot.previousRunId = failed.id; enrichedContextSnapshot.forceFreshSession = true; } } const caseRequest = opts.durableCaseHandoffRequest; const waitForCaseAdmission = (reason: string, message: string): never => { // The case ledger is the outbox. Roll back this admission attempt rather // than consume its one receipt ID in a generic skipped/deferred queue. throw conflict(message, { code: "case_handoff_admission_wait", reason }); }; let caseReceipt: { id: string; requestedAt: Date } | undefined; if (caseRequest) { if (opts.taskRetryRequestId || opts.durableChatRequest || opts.durableDotRequest || opts.durableMuseRequest || source !== "assignment" || reason !== "case_handoff") throw conflict("Invalid case handoff wake authority"); const [handoff] = await db.select().from(caseHandoffs).where(and(eq(caseHandoffs.id, caseRequest.handoffId), eq(caseHandoffs.companyId, agent.companyId), eq(caseHandoffs.issueId, caseRequest.issueId), eq(caseHandoffs.toAgentId, agentId), eq(caseHandoffs.wakeupRequestId, caseRequest.wakeupRequestId), inArray(caseHandoffs.state, ["pending_dispatch", "delivered"]))).limit(1); if (!handoff || issueId !== handoff.issueId || caseRequest.expectedIssueStatusVersion !== handoff.appliedIssueStatusVersion) throw conflict("Case handoff no longer matches its task assignment"); caseReceipt = { id: handoff.wakeupRequestId, requestedAt: handoff.createdAt }; opts = { ...opts, allowRunCoalescing: false, idempotencyKey: `case-handoff:${handoff.id}`, issueStateGuard: { assigneeAgentId: agentId, statuses: ["todo", "in_progress"], statusVersion: caseRequest.expectedIssueStatusVersion } }; } if (durableRequest) { assertDurableChatWakeupRequest(durableRequest, { agentId, companyId: agent.companyId, issueId, commentId: wakeCommentId ?? null, requestedByActorType: opts.requestedByActorType, requestedByActorId: opts.requestedByActorId, }); opts = { ...opts, idempotencyKey: durableRequest.idempotencyKey }; } if ( Object.hasOwn(enrichedContextSnapshot, "chatFailedRunRetry") && !durableRequest?.failedRunRetry ) { throw new FailedChatRunRetryAuthorizationError(); } if (durableRequest?.failedRunRetry) { opts = { ...opts, allowRunCoalescing: false }; } const dotRequest = opts.durableDotRequest; const museRequest = opts.durableMuseRequest; if (dotRequest && (durableRequest || museRequest || dotRequest.agentId !== agentId || dotRequest.companyId !== agent.companyId || dotRequest.issueId !== issueId || dotRequest.requestId !== payload?.dotRequestId || source !== "assignment" || opts.requestedByActorType !== "agent" || opts.requestedByActorId !== agentId || agent.adapterType !== "paperclip_runner" || !["openai_dot", "grok_bot"].includes(String(parseObject(agent.adapterConfig).provider)) || opts.idempotencyKey !== dotRequest.idempotencyKey)) { throw conflict("Dot work request does not match its admission authority."); } if (museRequest && (durableRequest || dotRequest || museRequest.agentId !== agentId || museRequest.companyId !== agent.companyId || museRequest.issueId !== issueId || museRequest.requestId !== payload?.museRequestId || source !== "assignment" || opts.requestedByActorType !== "agent" || opts.requestedByActorId !== agentId || agent.adapterType !== "paperclip_runner" || parseObject(agent.adapterConfig).provider !== "muse" || parseObject(agent.adapterConfig).museBindingId !== museRequest.bindingId || !Number.isSafeInteger(museRequest.bindingGeneration) || museRequest.bindingGeneration < 1 || payload?.museBindingId !== museRequest.bindingId || payload?.museBindingGeneration !== museRequest.bindingGeneration || payload?.museRequestDigest !== museRequest.requestDigest || !/^sha256:[a-f0-9]{64}$/.test(museRequest.requestDigest) || opts.idempotencyKey !== museRequest.idempotencyKey)) { throw conflict("Muse work request does not match its admission authority."); } const receiptRequest = durableRequest ?? dotRequest ?? museRequest ?? caseReceipt; const durableReceiptFields = receiptRequest ? { id: receiptRequest.id, requestedAt: receiptRequest.requestedAt } : {}; const existingDurableReceipt = async (queryDb: Db) => { if (!receiptRequest) return null; const receipt = await queryDb .select() .from(agentWakeupRequests) .where(eq(agentWakeupRequests.id, receiptRequest.id)) .limit(1) .then((rows) => rows[0] ?? null); if (receipt && durableRequest) assertDurableChatWakeupReceipt(durableRequest, receipt); if (receipt && caseRequest && (receipt.companyId !== agent.companyId || receipt.agentId !== agentId || receipt.payload?.issueId !== caseRequest.issueId || receipt.idempotencyKey !== `case-handoff:${caseRequest.handoffId}`)) throw conflict("Case handoff wake receipt identity changed"); if (receipt && dotRequest && (receipt.companyId !== dotRequest.companyId || receipt.agentId !== dotRequest.agentId || receipt.source !== "assignment" || receipt.requestedByActorType !== "agent" || receipt.requestedByActorId !== dotRequest.agentId || receipt.idempotencyKey !== dotRequest.idempotencyKey || receipt.payload?.issueId !== dotRequest.issueId || receipt.payload?.dotRequestId !== dotRequest.requestId)) { throw conflict("requestId was reused for another task."); } if (receipt && museRequest && (receipt.companyId !== museRequest.companyId || receipt.agentId !== museRequest.agentId || receipt.source !== "assignment" || receipt.requestedByActorType !== "agent" || receipt.requestedByActorId !== museRequest.agentId || receipt.idempotencyKey !== museRequest.idempotencyKey || receipt.payload?.issueId !== museRequest.issueId || receipt.payload?.museRequestId !== museRequest.requestId || receipt.payload?.museBindingId !== museRequest.bindingId || receipt.payload?.museBindingGeneration !== museRequest.bindingGeneration || receipt.payload?.museRequestDigest !== museRequest.requestDigest)) { throw conflict("Request ID reused with changed input."); } if (receipt && durableRequest?.failedRunRetry && opts.taskRetryRequestId && receipt.status === "deferred_issue_execution" && !receipt.runId && receipt.payload?.taskRetryRequestId === opts.taskRetryRequestId) { // A bound ordinary Retry may reconsider its same deferred chat receipt. // The issue lock, live chat authority and execution holds below still // decide whether it can dispatch; this never creates another wake. executionWaitRequestId = receipt.id; return null; } return receipt; }; const priorReceipt = await existingDurableReceipt(db); if (priorReceipt) { // Replaying an admission receipt is not fresh authority to dispatch it. // The normal queue owns dispatch and its current execution-policy checks. return priorReceipt.runId ? getRun(priorReceipt.runId) : null; } const agentDebug = parseObject(parseObject(agent.runtimeConfig).debug); const runDebug = parseObject(enrichedContextSnapshot.debug); if ( agentDebug.providerTrace === "raw" && runDebug.providerTrace !== "raw" ) { enrichedContextSnapshot.debug = { ...runDebug, providerTrace: "raw", }; enrichedContextSnapshot.providerTraceRequestedBy = `agent:${agent.id}:debug-setting`; enrichedContextSnapshot.providerTraceRequestSource = "agent_debug_setting"; } // Automatic signals are replaceable; user input and interaction delivery // retain distinct durable receipts even when the same gate blocks them. const coalesceExecutionWait = opts.requestedByActorType === "system" && !receiptRequest && !wakeCommentId && queuedCommentIdsFromRunContext(enrichedContextSnapshot).length === 0 && !isInteractionResolutionWakePayload(payload ?? {}) && !hasInteractionContinuationWakeContext(enrichedContextSnapshot); const writeSkippedRequest = async ( skipReason: string, patch: Partial = {}, waitCondition?: Record, ) => { // A temporary refusal must leave the handoff retryable, without burning // its one durable receipt ID on a skipped wake. if (caseRequest) return { created: false }; if (executionWaitRequestId) { await db.update(agentWakeupRequests).set({ payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify({ reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active" ? "This task is paused. Resume it to send your saved message." : "Waiting for task execution to be enabled. Your message is saved."), })}::jsonb)`, updatedAt: new Date(), }).where(and(eq(agentWakeupRequests.id, executionWaitRequestId), eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"))); return { created: false }; } const request = { ...durableReceiptFields, companyId: agent.companyId, agentId, source, triggerDetail, reason: skipReason, payload, status: "skipped", requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, finishedAt: new Date(), ...patch, }; if (waitCondition && issueId && isUuidLike(issueId)) { const waitIssueId = issueId; return db.transaction(async (tx) => { await tx.execute(sql`select id from issues where id = ${waitIssueId} and company_id = ${agent.companyId} for update`); return recordExecutionWait(tx as unknown as Db, { issueId: waitIssueId, request, condition: waitCondition, coalesce: coalesceExecutionWait, }); }); } await db.insert(agentWakeupRequests).values(request); return { created: true }; }; const writeSkippedHeartbeatRequest = async ( skipReason: string, details: Record, ) => { await writeSkippedRequest(skipReason, { payload: { ...(payload ?? {}), heartbeatSkip: details, }, }); }; const schedulingSuppression = await getSchedulingSuppression(); // A task drain holds ADMISSION, not the request. The drain is a // process-local pre-restart hold, so a wake that arrives while it is // active still names real work that must run once the process comes // back: leave it in the durable queue and let the dispatch-side checks // (startNextQueuedRunForAgent / executeRun) keep it from starting until // the drain lifts or the restart clears it. Writing it as `skipped` // here dropped the wake permanently — an accepted plan whose // continuation wake landed mid-drain left its issue in `todo` with no // run and no path until a person noticed. if ( schedulingSuppression.suppressed && schedulingSuppression.reason !== "task_drain" ) { await writeSkippedHeartbeatRequest("heartbeat.scheduling_suppressed", { reason: schedulingSuppression.reason, }); return null; } const worktreeExecutionCutoff = opts.requestedByActorType === "user" ? null : await getWorktreeExecutionCutoff(); const company = await db .select({ status: companies.status }) .from(companies) .where(eq(companies.id, agent.companyId)) .then((rows) => rows[0] ?? null); if (!company || company.status !== "active") { if (caseRequest) return null; const companyStatus = company?.status ?? "missing"; if (opts.requestedByActorType === "user") { throw conflict("Company is not active", { status: companyStatus }); } await writeSkippedRequest("company.inactive", { error: `Wake suppressed because company status is ${companyStatus}`, }, { companyStatus }); return null; } const explicitResumeSession = await resolveExplicitResumeSessionOverride( agent, payload, taskKey, ); if (explicitResumeSession) { enrichedContextSnapshot.resumeFromRunId = explicitResumeSession.resumeFromRunId; enrichedContextSnapshot.resumeSessionDisplayId = explicitResumeSession.sessionDisplayId; enrichedContextSnapshot.resumeSessionParams = explicitResumeSession.sessionParams; if ( !readNonEmptyString(enrichedContextSnapshot.issueId) && explicitResumeSession.issueId ) { enrichedContextSnapshot.issueId = explicitResumeSession.issueId; } if ( !readNonEmptyString(enrichedContextSnapshot.taskId) && explicitResumeSession.taskId ) { enrichedContextSnapshot.taskId = explicitResumeSession.taskId; } if ( !readNonEmptyString(enrichedContextSnapshot.taskKey) && explicitResumeSession.taskKey ) { enrichedContextSnapshot.taskKey = explicitResumeSession.taskKey; } issueId = readNonEmptyString(enrichedContextSnapshot.issueId) ?? issueId; } const effectiveTaskKey = readNonEmptyString(enrichedContextSnapshot.taskKey) ?? taskKey; const sessionBefore = explicitResumeSession?.sessionDisplayId ?? (await resolveSessionBeforeForWakeup(agent, effectiveTaskKey)); let hasResolvablePriorSessionWorkspace: boolean | null = null; const resolveHasResolvablePriorSessionWorkspace = async () => { if (hasResolvablePriorSessionWorkspace !== null) return hasResolvablePriorSessionWorkspace; hasResolvablePriorSessionWorkspace = issueId ? await hasResolvablePriorSessionWorkspaceForWake({ agent, contextSnapshot: enrichedContextSnapshot, taskKey: effectiveTaskKey, explicitResumeSession, }) : false; return hasResolvablePriorSessionWorkspace; }; const continuationAttempt = readContinuationAttempt( enrichedContextSnapshot.livenessContinuationAttempt, ); let projectId = readNonEmptyString(enrichedContextSnapshot.projectId); if (!projectId && issueId) { // Look up by either UUID or identifier (e.g. "ENV-13"), but always scope // by companyId so a row from another tenant can never be returned even // when identifiers collide across companies. Guard the UUID arm because // issues.id is a Postgres uuid column — passing "ENV-13" into eq(issues.id, …) // would fail with an invalid-input-syntax cast error before the OR is // evaluated. const lookupIsUuid = isUuidLike(issueId); const idMatch = lookupIsUuid ? or( eq(issues.id, issueId), eq(issues.identifier, issueId.toUpperCase()), ) : eq(issues.identifier, issueId.toUpperCase()); const resolvedIssue = await db .select({ id: issues.id, projectId: issues.projectId, createdAt: issues.createdAt, }) .from(issues) .where(and(eq(issues.companyId, agent.companyId), idMatch)) .then((rows) => rows[0] ?? null); if (resolvedIssue) { if ( worktreeExecutionCutoff && resolvedIssue.createdAt < worktreeExecutionCutoff ) { await writeSkippedHeartbeatRequest( "heartbeat.worktree_execution_cutoff", { reason: "worktree_execution_cutoff", cutoff: worktreeExecutionCutoff.toISOString(), issueId: resolvedIssue.id, }, ); return null; } projectId = resolvedIssue.projectId ?? null; // Canonicalize context to the UUID so downstream lookups always use UUID if (resolvedIssue.id !== issueId) { issueId = resolvedIssue.id; enrichedContextSnapshot.issueId = issueId; if (readNonEmptyString(enrichedContextSnapshot.taskId)) { enrichedContextSnapshot.taskId = issueId; } } } } // Propagate projectId into context so resolveWorkspaceForRun can bind the // project workspace even when context.projectId wasn't set by the caller. if (projectId && !readNonEmptyString(enrichedContextSnapshot.projectId)) { enrichedContextSnapshot.projectId = projectId; } const isolatedWorkspacesEnabled = issueId ? (await instanceSettings.getExperimental()).enableIsolatedWorkspaces : false; let operatorResponsibleUserId: string | null = opts.manualUserWake ? opts.requestedByActorId! : null; let queuedResponsibleUserIdPromise: Promise | null = null; const resolveQueuedResponsibleUserId = () => { if (operatorResponsibleUserId) return Promise.resolve(operatorResponsibleUserId); queuedResponsibleUserIdPromise ??= (async () => { const queuedIssueContext = issueId ? await getIssueExecutionContext(agent.companyId, issueId) : null; const queuedRoutineEnvContext = await getRoutineEnvForExecutionIssue( agent.companyId, queuedIssueContext, ); const queuedResponsibleUserId = await resolveResponsibleUserIdForRunSeed({ companyId: agent.companyId, contextSnapshot: enrichedContextSnapshot, issueContext: queuedIssueContext, routineEnvContext: queuedRoutineEnvContext, requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, source, triggerDetail, }); if (!queuedResponsibleUserId) { throw new HttpError( 422, "Unable to resolve responsible user for heartbeat run dispatch", { code: "responsible_user_unresolved", agentId, companyId: agent.companyId, issueId: issueId ?? null, source, triggerDetail, wakeReason: readNonEmptyString( enrichedContextSnapshot.wakeReason, ), }, ); } return queuedResponsibleUserId; })(); return queuedResponsibleUserIdPromise; }; const budgetBlock = await budgets.getInvocationBlock( agent.companyId, agentId, { issueId, projectId, }, ); if (budgetBlock) { await writeSkippedRequest("budget.blocked", { error: budgetBlock.reason }, { scopeType: budgetBlock.scopeType, scopeId: budgetBlock.scopeId, }); if (caseRequest) return null; throw conflict(budgetBlock.reason, { scopeType: budgetBlock.scopeType, scopeId: budgetBlock.scopeId, }); } // Setup delays execution, but must not discard work accepted by a caller. const invokability = await getAgentInvokability(isAgentAwaitingSetup(agent) ? { ...agent, status: "idle" } : agent); if (!invokability.invokable) { if (caseRequest) return null; if (opts.requestedByActorType !== "user" || executionWaitRequestId) { await writeSkippedRequest("agent.not_invokable", { error: invokability.message, }, { status: agent.status, reason: invokability.reason }); } throw conflict(invokability.message, { status: agent.status, reason: invokability.reason, invalidOrgChain: invokability.invalidOrgChain, ...invokability.details, }); } const policy = parseHeartbeatPolicy(agent); if (source === "timer" && !policy.enabled) { await writeSkippedRequest("heartbeat.disabled", {}, { enabled: false }); return null; } if (source !== "timer" && !policy.wakeOnDemand) { await writeSkippedRequest("heartbeat.wakeOnDemand.disabled", {}, { wakeOnDemand: false }); return null; } const genericTimerWake = source === "timer" && !issueId && !wakeCommentId && !readNonEmptyString(enrichedContextSnapshot.taskId) && !readNonEmptyString(enrichedContextSnapshot.taskKey); if ( policy.skipTimerWhenNoActionableWork && genericTimerWake && !(await hasActionableTimerWork(agent)) ) { await writeSkippedHeartbeatRequest("heartbeat.timer.no_actionable_work", { reason: "No assigned todo or in_progress issue requires this agent before timer adapter invocation.", }); await markTimerHeartbeatChecked(agentId, source); return null; } if (issueId) { const activePauseHold = await treeControlSvc.getActivePauseHoldGate( agent.companyId, issueId, ); if (activePauseHold) { const treeHoldInteractionWake = await isVerifiedIssueTreeControlInteractionWake(db, { companyId: agent.companyId, issueId, agentId, contextSnapshot: enrichedContextSnapshot, requestedByActorType: opts.requestedByActorType, requestedByActorId: opts.requestedByActorId, }); if (!treeHoldInteractionWake) { const wait = await writeSkippedRequest("issue_tree_hold_active", {}, { holdId: activePauseHold.holdId, }); if (wait.created) await logActivity(db, { companyId: agent.companyId, actorType: "system", actorId: "system", agentId, runId: null, action: "issue.tree_hold_wakeup_deferred", entityType: "issue", entityId: issueId, details: { holdId: activePauseHold.holdId, rootIssueId: activePauseHold.rootIssueId, requestedReason: reason, source, triggerDetail, securityPrinciples: [ "Complete Mediation", "Fail Securely", "Secure Defaults", ], }, }); return null; } enrichedContextSnapshot.treeHoldInteraction = true; enrichedContextSnapshot.activeTreeHold = { holdId: activePauseHold.holdId, rootIssueId: activePauseHold.rootIssueId, mode: activePauseHold.mode, reason: activePauseHold.reason, releasePolicy: activePauseHold.releasePolicy, interaction: true, }; } } if (issueId) { // Mention-triggered wakes can request input from another agent, but they must // still respect the issue execution lock so a second agent cannot start on the // same issue workspace while the assignee already has a live run. const agentNameKey = normalizeAgentNameKey(agent.name); const cancelledRunsToEmit: (typeof heartbeatRuns.$inferSelect)[] = []; const outcome = await db .transaction(async (tx) => { await tx.execute( sql`select id from issues where id = ${issueId} and company_id = ${agent.companyId} for update`, ); if (executionWaitRequestId) { const [pending] = await tx.select().from(agentWakeupRequests).where(and( eq(agentWakeupRequests.id, executionWaitRequestId), eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), // A user message can join a queue originally created by a // system wake. Admission validates the saved user comment or board click. (opts.queuedCommentInterruptId ?? opts.queuedCommentRequestId) === executionWaitRequestId ? undefined : eq(agentWakeupRequests.requestedByActorType, durableRequest?.failedRunRetry && opts.taskRetryRequestId ? opts.requestedByActorType ?? "" : "user"), opts.queuedCommentInterruptId === executionWaitRequestId ? sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt'->>'actorId' = ${opts.requestedByActorId ?? ""}` : opts.queuedCommentRequestId === executionWaitRequestId ? undefined : eq(agentWakeupRequests.requestedByActorId, opts.requestedByActorId ?? ""), sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, )); // The issue lock serializes cleanup callbacks and periodic workers. // An adopted, discarded, or edited receipt is no longer authority. if (!pending || (!opts.taskRetryRequestId && !(wakeCommentId && queuedCommentIdsFromWakePayload(pending.payload).includes(wakeCommentId)) && !(opts.queuedCommentInterruptId && await readQueuedInteractionResponse(tx as unknown as Db, agent.companyId, issueId, pending.payload)))) { return { kind: "deferred" as const }; } if (opts.taskRetryRequestId && durableRequest?.failedRunRetry) { if (pending.id !== durableRequest.id || pending.payload?.taskRetryRequestId !== opts.taskRetryRequestId) return { kind: "deferred" as const }; // The parent board Retry is checked separately below after the // connected conversation reauthorizes this exact child receipt. } else if (opts.taskRetryRequestId) { const retry = readTaskRetryIntent(pending.payload); if (pending.id !== opts.taskRetryRequestId || retry?.generationId !== pending.id || retry.chatActionId || retry.actorId !== opts.requestedByActorId || retry.sourceRunId !== enrichedContextSnapshot.previousRunId) return { kind: "deferred" as const }; const permission = await authorizationService(tx as unknown as Db).decide({ actor: { type: "board", userId: retry.actorId, source: retry.actorSource }, action: "issue:mutate", resource: { type: "issue", companyId: agent.companyId, issueId }, }); if (!permission.allowed) return { kind: "deferred" as const }; } if (opts.queuedCommentRequestId) { const ids = await undeliveredLegacyUserCommentIds(tx as unknown as Db, agent.companyId, issueId, agentId, queuedCommentIdsFromWakePayload(pending.payload)); if (!wakeCommentId || !ids.includes(wakeCommentId)) return { kind: "deferred" as const }; pending.payload = withQueuedCommentIdsInWakePayload(parseObject(pending.payload), ids); await tx.update(agentWakeupRequests).set({ payload: pending.payload }).where(and( eq(agentWakeupRequests.id, pending.id), eq(agentWakeupRequests.companyId, agent.companyId), )); } if (!opts.queuedCommentInterruptId && !opts.queuedCommentRequestId && pending.payload?.manualUserWake === true) { // A persisted manual wake keeps its actor when an execution wait // resumes. The locked receipt above has revalidated that actor. payload = { ...payload, manualUserWake: true }; operatorResponsibleUserId = opts.requestedByActorId!; } if (opts.queuedCommentInterruptId) { // The locked board receipt supplies execution authority even when // another user authored the messages. Dispatch revalidates the receipt. operatorResponsibleUserId = opts.requestedByActorId!; } if (opts.queuedCommentInterruptId || opts.queuedCommentRequestId) { // Edits/discards between the click and dispatch remain authoritative. Object.assign(enrichedContextSnapshot, withQueuedCommentIdsInRunContext( enrichedContextSnapshot, queuedCommentIdsFromWakePayload(pending.payload), )); } } let automaticParentRunId: string | null = null; if ( source === "automation" && opts.requestedByActorType === "system" ) { const nativeParent = await readChatControlNativeParent( tx as unknown as Db, { companyId: agent.companyId, issueId, agentId }, opts.requestedByActorId ?? null, ); const genericParent = !opts.requestedByActorId && reason === "issue_continuation_needed" ? readNonEmptyString(enrichedContextSnapshot.retryOfRunId) : null; automaticParentRunId = nativeParent.kind === "parent" ? nativeParent.runId : genericParent; if (automaticParentRunId) { const [parent] = await tx .select({ agentId: heartbeatRuns.agentId, context: heartbeatRuns.contextSnapshot, }) .from(heartbeatRuns) .where( and( eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.id, automaticParentRunId), ), ) .limit(1); if ( parent && (parseObject(parent.context).issueId ?? parseObject(parent.context).taskId) === issueId && parent.agentId !== agentId ) automaticParentRunId = null; } const proof = nativeParent.kind === "unresolved" ? { kind: "unresolved" as const } : automaticParentRunId ? await readChatControlRecoveryStop( tx as unknown as Db, { companyId: agent.companyId, issueId, agentId, sourceRunId: automaticParentRunId, }, true, ) : { kind: "clear" as const }; if (proof.kind !== "clear") { await tx .insert(agentWakeupRequests) .values({ companyId: agent.companyId, agentId, source, triggerDetail, reason: proof.kind === "stopped" ? CHAT_CONTROL_RECOVERY_STOP_CODE : CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE, payload: { issueId }, status: "skipped", requestedByActorType: "system", requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, finishedAt: new Date(), }); return { kind: "skipped" as const }; } } const durableReceipt = await existingDurableReceipt( tx as unknown as Db, ); if (durableReceipt) return { kind: "durable" as const, receipt: durableReceipt }; if (caseRequest) { const [handoff] = await tx.select().from(caseHandoffs).where(and(eq(caseHandoffs.id, caseRequest.handoffId), eq(caseHandoffs.companyId, agent.companyId), eq(caseHandoffs.issueId, caseRequest.issueId), eq(caseHandoffs.state, "pending_dispatch"), eq(caseHandoffs.toAgentId, agentId), eq(caseHandoffs.wakeupRequestId, caseRequest.wakeupRequestId))).for("update"); if (!handoff || handoff.appliedIssueStatusVersion !== caseRequest.expectedIssueStatusVersion) throw conflict("Case handoff delivery was superseded"); } const failedChatRetry = await authorizeFailedChatRunRetryWake( db, tx as unknown as Db, { phase: "admission", wakeupRequestId: durableRequest?.id ?? null, companyId: agent.companyId, agentId, issueId, contextSnapshot: enrichedContextSnapshot, }, ); if ( failedChatRetry !== Boolean(durableRequest?.failedRunRetry) || (failedChatRetry && durableRequest?.failedRunRetry?.failedRunId !== readNonEmptyString(enrichedContextSnapshot.retryOfRunId)) ) { throw new FailedChatRunRetryAuthorizationError(); } if (durableRequest) { if (issueId !== durableRequest.issueId) throw new Error("chat_inbound_wakeup_binding_denied"); await durableRequest.authorize(tx as unknown as Db); if (opts.taskRetryRequestId) { const [retryWake] = await tx.select().from(agentWakeupRequests).where(and( eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.id, opts.taskRetryRequestId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), )).for("update"); const retry = readTaskRetryIntent(retryWake?.payload); if (!failedChatRetry || retry?.chatActionId !== durableRequest.id || retry.actorId !== taskRetryActorId || retry.sourceRunId !== durableRequest.failedRunRetry?.failedRunId) throw new FailedChatRunRetryAuthorizationError(); const permission = await authorizationService(tx as unknown as Db).decide({ actor: { type: "board", userId: retry.actorId, source: retry.actorSource }, action: "issue:mutate", resource: { type: "issue", companyId: agent.companyId, issueId }, }); if (!permission.allowed) return { kind: "deferred" as const }; } } else { if ( source === "on_demand" && triggerDetail === "manual" && reason === "retry_failed_run" ) { // Generic Board retry does not carry server-authorized failed-run // lineage. Caller-supplied source/comment/retry markers cannot // restore it, including for a retired conversation generation. const [chatBinding] = await tx .select({ id: chatConversations.id }) .from(chatConversations) .innerJoin(chatEndpoints, eq(chatEndpoints.id, chatConversations.endpointId)) .where( and( eq(chatEndpoints.externalExecutionPolicy, "restricted"), eq(chatConversations.companyId, agent.companyId), eq(chatConversations.issueId, issueId), ), ) .limit(1); if (chatBinding) { throw conflict( "Retry needs the exact failed chat request and current access. Send the request again in the current connected conversation.", { code: "chat_failed_run_retry_requires_authorized_context" }, ); } } const [held] = await tx .select({ id: issues.id }) .from(issues) .where( and( eq(issues.id, issueId), eq(issues.companyId, agent.companyId), unadmittedChatWakeupCondition(issues.id, issues.companyId), ), ) .limit(1); if (held) { throw conflict( "This task has external chat input that has not been admitted. Check the connection's Activity; let pending input finish, or restore access and send a new authorized message before starting the task.", { code: "chat_inbound_wakeup_unadmitted", issueId }, ); } } const issue = await tx .select({ id: issues.id, companyId: issues.companyId, identifier: issues.identifier, conversationAgentId: issues.conversationAgentId, conversationUserId: issues.conversationUserId, conversationState: issues.conversationState, status: issues.status, statusVersion: issues.statusVersion, projectId: issues.projectId, projectWorkspaceId: issues.projectWorkspaceId, executionWorkspaceId: issues.executionWorkspaceId, executionWorkspacePreference: issues.executionWorkspacePreference, executionWorkspaceSettings: issues.executionWorkspaceSettings, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, monitorNextCheckAt: issues.monitorNextCheckAt, monitorWakeRequestedAt: issues.monitorWakeRequestedAt, executionRunId: issues.executionRunId, executionAgentNameKey: issues.executionAgentNameKey, createdAt: issues.createdAt, }) .from(issues) .where( and( eq(issues.id, issueId), eq(issues.companyId, agent.companyId), ), ) .then((rows) => rows[0] ?? null); if (!issue) { if (caseRequest) throw conflict("Case handoff task no longer exists"); await tx.insert(agentWakeupRequests).values({ ...durableReceiptFields, companyId: agent.companyId, agentId, source, triggerDetail, reason: "issue_execution_issue_not_found", payload, status: "skipped", requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, finishedAt: new Date(), }); return { kind: "skipped" as const }; } if (taskRetryActorId) { await assertTaskRetryConversationAllowed(tx as unknown as Db, issue, taskRetryActorId); if (await retireSupersededTaskRetry(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id, requestId: opts.taskRetryRequestId!, sourceRunId: readNonEmptyString(enrichedContextSnapshot.previousRunId)!, })) return { kind: "deferred" as const }; } const issueStateGuard = opts.issueStateGuard; const activeMonitorRun = issueStateGuard?.monitorNextCheckAt === undefined ? null : await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, issue.companyId), eq(heartbeatRuns.nativeIssueId, issue.id), eq(heartbeatRuns.runtimeMode, "native"), inArray(heartbeatRuns.status, ["queued", "running", "scheduled_retry"]), )).limit(1).then(rows => rows[0] ?? null); if ( issueStateGuard && (!issueStateGuard.statuses.includes(issue.status) || issue.assigneeAgentId !== issueStateGuard.assigneeAgentId || (issueStateGuard.statusVersion !== undefined && issue.statusVersion !== issueStateGuard.statusVersion) || (issueStateGuard.monitorNextCheckAt !== undefined && ( activeMonitorRun !== null || issue.assigneeUserId !== null || issue.monitorNextCheckAt?.toISOString() !== issueStateGuard.monitorNextCheckAt || issue.monitorWakeRequestedAt?.toISOString() !== issueStateGuard.monitorWakeRequestedAt ))) ) { if (caseRequest) throw conflict("Case handoff task changed before admission"); // A deferred monitor retains its schedule; do not create a receipt // that could suppress its next admission attempt. if (issueStateGuard.monitorNextCheckAt !== undefined) return { kind: "skipped" as const }; await tx.insert(agentWakeupRequests).values({ ...durableReceiptFields, companyId: agent.companyId, agentId, source, triggerDetail, reason: "issue_state_guard_mismatch", payload: { ...(payload ?? {}), heartbeatSkip: { reason: "Issue status or assignee changed before the wake could be queued.", issueId: issue.id, expectedStatuses: issueStateGuard.statuses, actualStatus: issue.status, expectedAssigneeAgentId: issueStateGuard.assigneeAgentId, actualAssigneeAgentId: issue.assigneeAgentId, }, }, status: "skipped", requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, finishedAt: new Date(), }); return { kind: "skipped" as const }; } if (opts.failedRunId) { // The issue lock makes double-clicks and network retries adopt the // same successor, including after it has already finished. const [previousRetry] = await tx.select().from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, issue.companyId), eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.retryOfRunId, opts.failedRunId), sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'retry_failed_run'`, )).orderBy(desc(heartbeatRuns.createdAt)).limit(1); if (previousRetry) return { kind: "replayed" as const, run: previousRetry }; } let reconciledSourceRunId: string | null = null; let reconciledRestoreRetryCount: number | null = null; if (executionReconciliationWake) { const actionId = readNonEmptyString( enrichedContextSnapshot.recoveryActionId, ); if ( !actionId || !isUuidLike(actionId) || source !== "automation" || triggerDetail !== "system" || reason !== "issue_recovery_action_restored" || opts.requestedByActorType !== "system" || opts.requestedByActorId !== "execution-recovery" || opts.idempotencyKey !== `execution-reconciliation:${actionId}` || enrichedContextSnapshot.source !== "execution.reconciled" || enrichedContextSnapshot.forceFreshSession !== true || payload?.issueId !== issue.id || payload?.recoveryActionId !== actionId || issue.assigneeAgentId !== agentId || ["done", "cancelled"].includes(issue.status) ) return { kind: "skipped" as const }; // The issue lock serializes all admissions for this source. Validate // the durable operator decision, then reconcile a prior queue commit // before considering a new wake (including a now-terminal successor). const [action] = await tx .select() .from(issueRecoveryActions) .where( and( eq(issueRecoveryActions.companyId, issue.companyId), eq(issueRecoveryActions.sourceIssueId, issue.id), eq(issueRecoveryActions.id, actionId), ), ) .for("update"); const decision = parseObject( action?.evidence.executionReconciliation, ); const sourceRunId = readNonEmptyString(decision.runId); if ( !action || action.status !== "resolved" || action.kind !== "active_run_watchdog" || action.returnOwnerAgentId !== agentId || !sourceRunId || !isUuidLike(sourceRunId) || decision.providerStopped !== true || !["completed", "not_performed", "mixed"].includes( String(decision.actionOutcome), ) || !readNonEmptyString(decision.outcomeEvidence) || enrichedContextSnapshot.previousRunId !== sourceRunId || enrichedContextSnapshot.retryOfRunId !== sourceRunId || !["pending", "delivered"].includes( String(action.evidence.continuationDelivery), ) ) return { kind: "skipped" as const }; const [existingWake] = await tx .select() .from(agentWakeupRequests) .where( and( eq(agentWakeupRequests.companyId, issue.companyId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.idempotencyKey, opts.idempotencyKey), ne(agentWakeupRequests.status, "skipped"), ), ) .orderBy(asc(agentWakeupRequests.requestedAt)) .limit(1); if (existingWake) { if ( existingWake.payload?.issueId !== issue.id || existingWake.payload?.recoveryActionId !== action.id || existingWake.requestedByActorType !== "system" || existingWake.requestedByActorId !== "execution-recovery" || !existingWake.runId ) return { kind: "deferred" as const }; const [existingRun] = await tx .select() .from(heartbeatRuns) .where( and( eq(heartbeatRuns.companyId, issue.companyId), eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.id, existingWake.runId), ), ); if ( !existingRun || existingRun.contextSnapshot?.issueId !== issue.id || existingRun.contextSnapshot?.recoveryActionId !== action.id || existingRun.contextSnapshot?.previousRunId !== sourceRunId ) return { kind: "deferred" as const }; return { kind: "replayed" as const, run: existingRun }; } if (action.evidence.continuationDelivery !== "pending") return { kind: "skipped" as const }; const [reconciledRun] = await tx.select().from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, issue.companyId), eq(heartbeatRuns.id, sourceRunId), )); if (decision.workspaceReconstruction) { await validateNativeWorkspaceReconstruction(tx as unknown as Db, { companyId: issue.companyId, issueId: issue.id, runId: sourceRunId, actionId, }); reconciledRestoreRetryCount = executionFailureRetryCount(reconciledRun!); enrichedContextSnapshot.workspaceReconstructionActionId = action.id; } if (hasWorkspaceRestoreFailure(reconciledRun?.resultJson)) { if ((readNonEmptyString(decision.workspaceRepairEvidence)?.length ?? 0) < 20) return { kind: "skipped" as const }; // Repair does not reset the remaining automatic retry budget. reconciledRestoreRetryCount = executionFailureRetryCount(reconciledRun!); } reconciledSourceRunId = sourceRunId; } let continuationWait = { reason: "execution_recovery", message: "Waiting for execution recovery. Your message is saved." }; const deferBlockedExecution = async ( executionBlocker: NonNullable>>, ) => { if (caseRequest) waitForCaseAdmission("execution_reconciliation_required", executionBlocker.nextAction); opts.issueStateGuard?.onExecutionDeferred?.(executionBlocker.cause); const condition = { recoveryActionId: executionBlocker.recoveryActionId, ...continuationWait }; if (executionWaitRequestId) { await tx.update(agentWakeupRequests).set({ payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', ${JSON.stringify(condition)}::jsonb)`, updatedAt: new Date(), }).where(eq(agentWakeupRequests.id, executionWaitRequestId)); return { kind: "deferred" as const }; } if (durableRequest || wakeCommentId || hasInteractionContinuationWakeContext(enrichedContextSnapshot) || readNonEmptyString(enrichedContextSnapshot.nativeStatusWakeIntentId)) { await tx.insert(agentWakeupRequests).values({ ...durableReceiptFields, companyId: agent.companyId, agentId, source, triggerDetail, reason, payload: withQueuedCommentIdsInWakePayload({ ...payload, issueId: issue.id, [DEFERRED_WAKE_CONTEXT_KEY]: enrichedContextSnapshot, executionWait: condition, }, [...new Set([ ...queuedCommentIdsFromRunContext(enrichedContextSnapshot), ...(wakeCommentId ? [wakeCommentId] : []), ])]), status: "deferred_issue_execution", requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, }); } else { await recordExecutionWait(tx as unknown as Db, { issueId: issue.id, condition, coalesce: coalesceExecutionWait, request: { ...durableReceiptFields, companyId: agent.companyId, agentId, source, triggerDetail, reason: "execution_reconciliation_required", error: executionBlocker.nextAction, payload, requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, }, }); } return { kind: "deferred" as const }; }; const explicitContinuationRunId = randomUUID(); const executionBlocker = await getExecutionBlocker( tx as unknown as Db, issue.companyId, issue.id, { conversationResetCommentId: opts.requestedByActorType === "user" ? wakeCommentId : null }, ); // Prove eligibility without retiring the hold. Later gates can still // decline this wake; hold retirement and successor creation stay atomic. // A bound chat request has already rechecked its current principal // above. Treat its new user message like a board comment, but keep // failed-run retry actions on their separate exact-request path. const ownsReconstructionContinuation = reconciledSourceRunId && executionBlocker?.runId === reconciledSourceRunId && executionBlocker.recoveryActionId === enrichedContextSnapshot.recoveryActionId; if (executionBlocker && !ownsReconstructionContinuation && !(await admitExplicitNativeContinuation({ db: tx as unknown as Db, companyId: issue.companyId, issueId: issue.id, agentId, actorType: taskRetryActorId ? "user" : opts.requestedByActorType, actorId: taskRetryActorId ?? opts.requestedByActorId, reason: taskRetryActorId ? "task_retry" : durableRequest && !failedChatRetry ? "issue_commented" : reason, commentId: wakeCommentId ?? null, failedRunId: opts.failedRunId, successorRunId: explicitContinuationRunId, queuedCommentInterruptId: opts.queuedCommentInterruptId, queuedCommentRequestId: opts.queuedCommentRequestId, taskRetryRequestId: opts.taskRetryRequestId, dryRun: true, onBlocked: (reason, message) => { continuationWait = { reason, message }; }, }))) return deferBlockedExecution(executionBlocker); if ( worktreeExecutionCutoff && issue.createdAt < worktreeExecutionCutoff ) { if (caseRequest) waitForCaseAdmission("heartbeat.worktree_execution_cutoff", "Case handoff is waiting for workspace execution to be enabled"); await tx.insert(agentWakeupRequests).values({ ...durableReceiptFields, companyId: agent.companyId, agentId, source, triggerDetail, reason: "heartbeat.worktree_execution_cutoff", payload: { ...(payload ?? {}), heartbeatSkip: { reason: "worktree_execution_cutoff", cutoff: worktreeExecutionCutoff.toISOString(), issueId: issue.id, }, }, status: "skipped", requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, finishedAt: new Date(), }); return { kind: "skipped" as const }; } const cancelStaleScheduledRetry = async ( scheduledRun: typeof heartbeatRuns.$inferSelect, ) => { const issueCancelled = issue.status === "cancelled"; if ( scheduledRun.status !== "scheduled_retry" || (scheduledRun.agentId === issue.assigneeAgentId && !issueCancelled) ) { return false; } const now = new Date(); const reason = issueCancelled ? "Cancelled because the issue was cancelled before the scheduled retry became due" : "Cancelled because the issue was reassigned before the scheduled retry became due"; const cancelled = await tx .update(heartbeatRuns) .set({ status: "cancelled", finishedAt: now, error: reason, errorCode: issueCancelled ? "issue_cancelled" : "issue_reassigned", updatedAt: now, }) .where( and( eq(heartbeatRuns.id, scheduledRun.id), eq(heartbeatRuns.status, "scheduled_retry"), ), ) .returning() .then((rows) => rows[0] ?? null); if (!cancelled) return false; if (scheduledRun.wakeupRequestId) { await tx .update(agentWakeupRequests) .set({ status: "cancelled", finishedAt: now, error: reason, updatedAt: now, }) .where( eq(agentWakeupRequests.id, scheduledRun.wakeupRequestId), ); } if (issue.executionRunId === scheduledRun.id) { await tx .update(issues) .set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, updatedAt: now, }) .where( and( eq(issues.id, issue.id), eq(issues.executionRunId, scheduledRun.id), ), ); } const eventSeq = await allocateHeartbeatRunEventSeq( tx as unknown as Db, cancelled.id, ); await tx.insert(heartbeatRunEvents).values({ companyId: cancelled.companyId, runId: cancelled.id, agentId: cancelled.agentId, seq: eventSeq, eventType: "lifecycle", stream: "system", level: "warn", message: issueCancelled ? "Scheduled retry cancelled because issue was cancelled before it became due" : "Scheduled retry cancelled because issue ownership changed before it became due", payload: { issueId: issue.id, issueStatus: issue.status, scheduledRetryAttempt: cancelled.scheduledRetryAttempt, scheduledRetryAt: cancelled.scheduledRetryAt ? new Date(cancelled.scheduledRetryAt).toISOString() : null, scheduledRetryReason: cancelled.scheduledRetryReason, previousRetryAgentId: cancelled.agentId, currentAssigneeAgentId: issue.assigneeAgentId, }, }); await tx .update(heartbeatRuns) .set({ nextEventSeq: eventSeq + 1, updatedAt: now }) .where(eq(heartbeatRuns.id, cancelled.id)); cancelledRunsToEmit.push(cancelled); return true; }; let activeExecutionRun = issue.executionRunId ? await tx .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, issue.executionRunId)) .then((rows) => rows[0] ?? null) : null; if ( activeExecutionRun && !EXECUTION_PATH_HEARTBEAT_RUN_STATUSES.includes( activeExecutionRun.status as (typeof EXECUTION_PATH_HEARTBEAT_RUN_STATUSES)[number], ) ) { activeExecutionRun = null; } if ( activeExecutionRun && (await cancelStaleScheduledRetry(activeExecutionRun)) ) { activeExecutionRun = null; } // A queued/scheduled run holding the lock for an agent that is // no longer the issue's assignee is stale by design — the issue // has been re-routed (e.g. blocked → in_review with a different // assignee). Cancel it and release the lock; otherwise the new // assignee's wake gets parked in `deferred_issue_execution` // forever, because the original queued holder will never run // (the issue's status / target now belongs to someone else). // // Race guard: pin the cancel UPDATE to the exact non-running // status we read above. A worker could transition the holder // from `queued` → `running` between the SELECT and this UPDATE; // the status predicate ensures we never clobber a freshly- // claimed running run. If zero rows matched, leave // `activeExecutionRun` populated so the defer path runs // normally against the now-running holder. if ( activeExecutionRun && activeExecutionRun.status !== "running" && issue.assigneeAgentId && activeExecutionRun.agentId !== issue.assigneeAgentId ) { const cancelled = await tx .update(heartbeatRuns) .set({ status: "cancelled", finishedAt: new Date(), error: "Execution lock released after issue reassigned to a different agent", errorCode: "lock_released_on_reassignment", updatedAt: new Date(), }) .where( and( eq(heartbeatRuns.id, activeExecutionRun.id), eq(heartbeatRuns.status, activeExecutionRun.status), ), ) .returning(); if (cancelled.length > 0) { cancelledRunsToEmit.push(cancelled[0]); if (activeExecutionRun.wakeupRequestId) { await tx .update(agentWakeupRequests) .set({ status: "cancelled", finishedAt: new Date(), error: "Execution lock released after issue reassigned to a different agent", updatedAt: new Date(), }) .where( eq( agentWakeupRequests.id, activeExecutionRun.wakeupRequestId, ), ); } activeExecutionRun = null; } } if (!activeExecutionRun && issue.executionRunId) { await tx .update(issues) .set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, updatedAt: new Date(), }) .where(eq(issues.id, issue.id)); } if (!activeExecutionRun) { const legacyRun = await tx .select() .from(heartbeatRuns) .where( and( eq(heartbeatRuns.companyId, issue.companyId), inArray(heartbeatRuns.status, [ ...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES, ]), sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`, ), ) .orderBy( sql`case when ${heartbeatRuns.status} = 'running' then 0 else 1 end`, asc(heartbeatRuns.createdAt), ) .limit(1) .then((rows) => rows[0] ?? null); if (legacyRun) { if (await cancelStaleScheduledRetry(legacyRun)) { activeExecutionRun = null; } else { activeExecutionRun = legacyRun; const legacyAgent = await tx .select({ name: agents.name }) .from(agents) .where(eq(agents.id, legacyRun.agentId)) .then((rows) => rows[0] ?? null); await tx .update(issues) .set({ executionRunId: legacyRun.id, executionAgentNameKey: normalizeAgentNameKey( legacyAgent?.name, ), executionLockedAt: new Date(), updatedAt: new Date(), }) .where(eq(issues.id, issue.id)); } } } const dependencyReadiness = await issuesSvc .listDependencyReadiness(issue.companyId, [issue.id], tx) .then((rows) => rows.get(issue.id) ?? null); // Blocked descendants should stay idle until the final blocker resolves. // Human comment/mention wakes are the exception: they may run in a // bounded interaction mode so the assignee can answer or triage. const blockedInteractionWake = dependencyReadiness && !dependencyReadiness.isDependencyReady && allowsIssueInteractionWake( enrichedContextSnapshot, ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS, ); if (blockedInteractionWake) { enrichedContextSnapshot.dependencyBlockedInteraction = true; enrichedContextSnapshot.unresolvedBlockerIssueIds = dependencyReadiness.unresolvedBlockerIssueIds; enrichedContextSnapshot.unresolvedBlockerCount = dependencyReadiness.unresolvedBlockerCount; enrichedContextSnapshot.unresolvedBlockerSummaries = await listUnresolvedBlockerSummaries( tx, issue.companyId, issue.id, dependencyReadiness.unresolvedBlockerIssueIds, ); } if ( !activeExecutionRun && dependencyReadiness && !dependencyReadiness.isDependencyReady && !blockedInteractionWake ) { if (caseRequest) waitForCaseAdmission("issue_dependencies_blocked", "Case handoff is waiting for task dependencies to resolve"); await recordExecutionWait(tx as unknown as Db, { issueId: issue.id, coalesce: coalesceExecutionWait, condition: { unresolvedBlockerIssueIds: [...dependencyReadiness.unresolvedBlockerIssueIds].sort() }, request: { ...durableReceiptFields, companyId: agent.companyId, agentId, source, triggerDetail, reason: "issue_dependencies_blocked", payload: { ...(payload ?? {}), issueId, unresolvedBlockerIssueIds: dependencyReadiness.unresolvedBlockerIssueIds, }, status: "skipped", requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, finishedAt: new Date(), }, }); return { kind: "skipped" as const }; } if ( isolatedWorkspacesEnabled && !activeExecutionRun && issue.status !== "done" && issue.status !== "cancelled" ) { const issueSettings = parseIssueExecutionWorkspaceSettings( issue.executionWorkspaceSettings, ); const resolvedMode = resolveExecutionWorkspaceMode({ projectPolicy: null, issueSettings, legacyUseProjectWorkspace: null, }); const workspaceManagedConfig = buildExecutionWorkspaceAdapterConfig( { agentConfig: parseObject(agent.adapterConfig), projectPolicy: null, issueSettings, mode: resolvedMode, legacyUseProjectWorkspace: null, }, ); const resolvedStrategy = resolveEffectiveWorkspaceStrategyType( resolvedMode, workspaceManagedConfig, ); const existingExecutionWorkspaceStatus = issue.executionWorkspaceId ? await tx .select({ status: executionWorkspaces.status }) .from(executionWorkspaces) .where( and( eq(executionWorkspaces.id, issue.executionWorkspaceId), eq(executionWorkspaces.companyId, issue.companyId), ), ) .then((rows) => rows[0]?.status ?? null) : null; const reuseRequest = resolveExecutionWorkspaceReuseRequestForIssue({ issueExecutionWorkspaceId: issue.executionWorkspaceId, issueExecutionWorkspacePreference: issue.executionWorkspacePreference, existingExecutionWorkspaceStatus, }); const hasResolvablePriorSessionWorkspace = await resolveHasResolvablePriorSessionWorkspace(); if ( isUnrunnableWorktreeCombo({ issue: { projectId: issue.projectId ?? projectId ?? null, projectWorkspaceId: issue.projectWorkspaceId, executionWorkspaceId: issue.executionWorkspaceId, executionWorkspacePreference: issue.executionWorkspacePreference, }, resolvedMode, resolvedStrategy, reusableExecutionWorkspaceAvailable: reuseRequest.existingExecutionWorkspaceAvailable, hasResolvablePriorSessionWorkspace, }) ) { if (caseRequest) waitForCaseAdmission(WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE); const now = new Date(); const issueLabel = formatIssueIdentifierLink( issue.identifier, issue.id, ); const blockedComment = [ `Paperclip blocked ${issueLabel} before dispatch because its workspace settings are not runnable.`, "", `- Code: \`${WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE}\``, `- Reason: ${WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE}`, `- Next action: ${WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION}`, ].join("\n"); await tx .update(issues) .set({ status: "blocked", checkoutRunId: null, executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, updatedAt: now, }) .where(eq(issues.id, issue.id)); await insertIssueComment(tx, { companyId: issue.companyId, issueId: issue.id, body: blockedComment, createdAt: now, updatedAt: now, }); await tx.insert(agentWakeupRequests).values({ ...durableReceiptFields, companyId: agent.companyId, agentId, source, triggerDetail, reason: WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, payload: { ...(payload ?? {}), issueId, heartbeatSkip: { code: WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, reason: WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE, remediation: WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION, }, }, status: "skipped", requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, finishedAt: now, }); await logActivity(tx as unknown as Db, { companyId: issue.companyId, actorType: "system", actorId: "system", agentId, runId: null, action: "issue.workspace_preflight_blocked", entityType: "issue", entityId: issue.id, details: { code: WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, reason: WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE, remediation: WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION, requestedReason: reason, source, triggerDetail, resolvedMode, resolvedStrategy, hasResolvablePriorSessionWorkspace, }, }); return { kind: "skipped" as const }; } } if (activeExecutionRun) { // The case ledger owns retry. Do not place its exact receipt in a // generic deferred queue that cannot retain handoff authority. if (caseRequest) return { kind: "deferred" as const }; // The resolved action is already a durable retry outbox. Do not merge // its fresh-session contract into unrelated work or create a second // deferred wake that could later replay the same reconciliation. if (reconciledSourceRunId) return { kind: "deferred" as const }; // Offer the input to the active external agent, but keep a durable queued wake // until it reads the comment. A finish racing the event tick must // leave unread input available for the next assignment. const followUpProvider = parseObject(agent.adapterConfig).provider; const museFollowUp = followUpProvider === "muse"; const bindingId = readNonEmptyString(parseObject(agent.adapterConfig)[museFollowUp ? "museBindingId" : followUpProvider === "grok_bot" ? "externalBindingId" : "dotBindingId"]); const followUpPayloadKey = museFollowUp ? "museAssignmentFollowUp" : "dotAssignmentFollowUp"; let followUp: false | { assignmentId: string; consumed: boolean } = false; let pendingFollowUps: typeof agentWakeupRequests.$inferSelect[] = []; if (agent.adapterType === "paperclip_runner" && ["openai_dot", "grok_bot", "muse"].includes(String(followUpProvider)) && bindingId && activeExecutionRun.agentId === agentId && issue.assigneeAgentId === agentId && reason === "issue_commented" && wakeCommentId && opts.allowRunCoalescing !== false && enrichedContextSnapshot.forceFreshSession !== true && !explicitResumeSession && !opts.manualUserWake && !enrichedContextSnapshot.interactionId && !payload?.interactionId && !receiptRequest) { // Receipt consumption uses the same issue -> wake -> run -> guard -> // binding order. Lock before publishing, then reuse these wake rows. pendingFollowUps = await tx.select().from(agentWakeupRequests).where(and( eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId), isNull(agentWakeupRequests.runId), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`, sql`${agentWakeupRequests.payload}->>${followUpPayloadKey} is not null`, )).orderBy(asc(agentWakeupRequests.createdAt), asc(agentWakeupRequests.id)).for("update"); const [currentRun] = await tx.select().from(heartbeatRuns).where(and( eq(heartbeatRuns.id, activeExecutionRun.id), eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "running"), eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.nativeIssueId, issue.id), )).for("update"); if (currentRun) followUp = await withExternalAdmissionGuard(tx, agent.companyId, agentId, () => (museFollowUp ? publishActiveMuseComment : publishActiveDotComment)(tx as unknown as Db, { companyId: agent.companyId, agentId, bindingId, runId: currentRun.id, issueId: issue.id, commentId: wakeCommentId, })); } if (followUp) { const assignmentId = followUp.assignmentId; const pending = followUp.consumed ? undefined : pendingFollowUps.find(wake => wake.payload?.[followUpPayloadKey] === assignmentId && wake.requestedByActorType === (opts.requestedByActorType ?? null) && wake.requestedByActorId === (opts.requestedByActorId ?? null)); if (pending) await tx.update(agentWakeupRequests).set({ payload: withQueuedCommentIdsInWakePayload(pending.payload, [...new Set([...queuedCommentIdsFromWakePayload(pending.payload), wakeCommentId!])]), coalescedCount: pending.coalescedCount + 1, updatedAt: new Date(), }).where(and(eq(agentWakeupRequests.id, pending.id), eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"))); else await tx.insert(agentWakeupRequests).values({ companyId: agent.companyId, agentId, source, triggerDetail, reason, payload: withQueuedCommentIdsInWakePayload({ ...payload, issueId: issue.id, [followUpPayloadKey]: followUp.assignmentId, [DEFERRED_WAKE_CONTEXT_KEY]: enrichedContextSnapshot }, [wakeCommentId!]), status: followUp.consumed ? "coalesced" : "deferred_issue_execution", runId: followUp.consumed ? activeExecutionRun.id : null, requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, finishedAt: followUp.consumed ? new Date() : null }); return { kind: "coalesced" as const, run: activeExecutionRun }; } const admissionScope = wakeQueue.createAdmissionTransactionScope( agent.companyId, tx as unknown as Db, ); const admission = await wakeQueue.admitWakeBehindIssueExecution( admissionScope, { companyId: agent.companyId, issueId: issue.id, agentId, agentNameKey, issueExecutionAgentNameKey: issue.executionAgentNameKey, activeExecutionRun: { id: activeExecutionRun.id, agentId: activeExecutionRun.agentId, status: activeExecutionRun.status, contextSnapshot: activeExecutionRun.contextSnapshot, wakeupRequestId: activeExecutionRun.wakeupRequestId, }, allowRunCoalescing: isConversation(issue) ? false : opts.allowRunCoalescing, durableReceipt: receiptRequest ? { id: receiptRequest.id, requestedAt: receiptRequest.requestedAt, } : undefined, reason, liveRunExecutions, wakeCommentId, forceFreshSession: enrichedContextSnapshot.forceFreshSession === true, contextSnapshot: enrichedContextSnapshot, source, triggerDetail, payload, requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, }, ); if (admission.kind === "coalesced") { return { kind: "coalesced" as const, run: admission.run as typeof heartbeatRuns.$inferSelect, }; } if (admission.kind === "deferred") { return { kind: "deferred" as const }; } // admission.kind === "proceed": no active run absorbed this wake, // so fall through to the ordinary queue path below. } // PAP-13775: no live run holds the lock, so this wake would start a // fresh adapter session. If this agent's recent runs on this issue // keep succeeding without any issue-visible progress and the wake // carries no new information, hold it back for an escalating cooldown // so external pollers/reconcilers can't storm full-price sessions. // Server-side recovery retries insert runs directly and never reach // this gate. A verified durable case receipt carries a new phase; // its agent-authored instruction comment is not an event-free wake. if ( !caseRequest && isThrottleCandidateIssueRewake({ reason, wakeCommentId: wakeCommentId ?? null, requestedByActorType: opts.requestedByActorType ?? null, forceFreshSession: enrichedContextSnapshot.forceFreshSession === true, hasExplicitResume: Boolean(explicitResumeSession), }) ) { const throttleNow = new Date(); const recentTerminalRuns = await tx .select({ id: heartbeatRuns.id, status: heartbeatRuns.status, finishedAt: heartbeatRuns.finishedAt, }) .from(heartbeatRuns) .where( and( eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agentId), sql`${heartbeatRuns.finishedAt} is not null`, gte( heartbeatRuns.finishedAt, new Date(throttleNow.getTime() - ISSUE_REWAKE_LOOKBACK_MS), ), sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`, ), ) .orderBy(desc(heartbeatRuns.finishedAt)) .limit(ISSUE_REWAKE_RUN_SAMPLE_LIMIT); if (recentTerminalRuns.length > 0) { const sampleRunIds = recentTerminalRuns.map( (sampleRun) => sampleRun.id, ); const progressRows = await tx .select({ runId: activityLog.runId }) .from(activityLog) .where( and( eq(activityLog.companyId, agent.companyId), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issue.id), inArray(activityLog.runId, sampleRunIds), inArray( activityLog.action, ISSUE_PROGRESS_ACTIVITY_ACTIONS, ), ), ); const lastRunFinishedAt = recentTerminalRuns[0]?.finishedAt ?? null; const newInputRows = lastRunFinishedAt ? await tx .select({ id: activityLog.id }) .from(activityLog) .where( and( eq(activityLog.companyId, agent.companyId), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issue.id), gt(activityLog.createdAt, lastRunFinishedAt), sql`coalesce(${activityLog.details}->>'origin', '') <> 'fast_response'`, inArray( activityLog.action, ISSUE_NEW_INPUT_ACTIVITY_ACTIONS, ), wakeCommentId && opts.requestedByActorType === "agent" ? ne(activityLog.actorType, "agent") : undefined, ), ) .limit(1) : []; const throttleDecision = evaluateIssueRewakeThrottle({ now: throttleNow, recentTerminalRuns, runIdsWithIssueProgress: new Set( progressRows .map((row) => row.runId) .filter((runId): runId is string => Boolean(runId)), ), // For an agent comment wake, the query excludes agent-authored // activity while preserving genuinely new user/system input. // Presentation/author metadata therefore cannot smuggle human // wake privilege, nor can it mask an actual human response. hasNewIssueInputSinceLastRun: newInputRows.length > 0, }); if (throttleDecision.blocked) { await tx.insert(agentWakeupRequests).values({ ...durableReceiptFields, companyId: agent.companyId, agentId, source, triggerDetail, reason: "issue_rewake_throttled", payload: { ...(payload ?? {}), issueId, heartbeatSkip: { reason: "issue_rewake_throttled", requestedReason: reason, noProgressStreak: throttleDecision.noProgressStreak, cooldownMs: throttleDecision.cooldownMs, lastRunFinishedAt: throttleDecision.lastRunFinishedAt.toISOString(), nextAllowedAt: throttleDecision.nextAllowedAt.toISOString(), }, }, status: "skipped", requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, finishedAt: throttleNow, }); return { kind: "skipped" as const }; } } } const dailyCapBlock = await getHeartbeatDailyCapBlock( agent, policy, {}, tx, ); if (dailyCapBlock) { if (caseRequest) waitForCaseAdmission(dailyCapBlock.reason, "Case handoff is waiting for the agent's daily limit to clear"); if (executionWaitRequestId && executionBlocker) { continuationWait = { reason: dailyCapBlock.reason, message: "The agent has reached its daily limit. Your message is saved until work can resume." }; return deferBlockedExecution(executionBlocker); } const now = new Date(); await tx.insert(agentWakeupRequests).values({ ...durableReceiptFields, companyId: agent.companyId, agentId, source, triggerDetail, reason: dailyCapBlock.reason, payload: { ...(payload ?? {}), heartbeatSkip: { reason: "Per-agent heartbeat daily cap reached before adapter invocation.", observed: dailyCapBlock.observed, limit: dailyCapBlock.limit, }, }, status: "skipped", requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, finishedAt: now, }); if (source === "timer") { await tx .update(agents) .set({ lastHeartbeatAt: now, updatedAt: now, }) .where(eq(agents.id, agentId)); } return { kind: "skipped" as const }; } const explicitContinuation = await admitExplicitNativeContinuation({ db: tx as unknown as Db, companyId: issue.companyId, issueId: issue.id, agentId, actorType: taskRetryActorId ? "user" : opts.requestedByActorType, actorId: taskRetryActorId ?? opts.requestedByActorId, reason: taskRetryActorId ? "task_retry" : durableRequest && !failedChatRetry ? "issue_commented" : reason, commentId: wakeCommentId ?? null, failedRunId: opts.failedRunId, successorRunId: explicitContinuationRunId, queuedCommentInterruptId: opts.queuedCommentInterruptId, queuedCommentRequestId: opts.queuedCommentRequestId, taskRetryRequestId: opts.taskRetryRequestId, }); if (!explicitContinuation && executionBlocker && !ownsReconstructionContinuation) return deferBlockedExecution(executionBlocker); if (explicitContinuation) { enrichedContextSnapshot.forceFreshSession = true; enrichedContextSnapshot.previousRunId = explicitContinuation.previousRunId; enrichedContextSnapshot.explicitUserContinuation = explicitContinuation; } const wakeupValues = { ...durableReceiptFields, companyId: agent.companyId, agentId, source, triggerDetail, reason, payload, status: "queued" as const, requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, }; // A deferred durable chat wake already owns this ID and source. // Promote that receipt in the same transaction as its successor. const wakeupRequest = durableRequest?.failedRunRetry && executionWaitRequestId === durableRequest.id ? await tx.update(agentWakeupRequests).set({ payload, status: "queued", error: null, finishedAt: null, updatedAt: new Date(), }).where(and(eq(agentWakeupRequests.id, durableRequest.id), eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.status, "deferred_issue_execution"), isNull(agentWakeupRequests.runId))) .returning().then(rows => rows[0]) : await tx.insert(agentWakeupRequests).values(wakeupValues).returning().then(rows => rows[0]); if (!wakeupRequest) return { kind: "deferred" as const }; // A handoff changes the executor, not the owner of saved user input. // Validate its exact stopped source while the issue row is locked; // unrelated agents and dedicated continuations keep their own wakes. const interruptedRunId = readNonEmptyString(enrichedContextSnapshot.interruptedRunId); const handoffSource = source === "assignment" && reason === "issue_assigned" && issue.assigneeAgentId === agentId && interruptedRunId && isUuidLike(interruptedRunId) ? await tx.select({ agentId: heartbeatRuns.agentId }).from(heartbeatRuns).where(and( eq(heartbeatRuns.id, interruptedRunId), eq(heartbeatRuns.companyId, issue.companyId), eq(heartbeatRuns.status, "cancelled"), eq(heartbeatRuns.errorCode, "issue_reassigned"), ne(heartbeatRuns.agentId, agentId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}`, or(isNull(heartbeatRuns.nativeIssueId), eq(heartbeatRuns.nativeIssueId, issue.id)), )).then(rows => rows[0] ?? null) : null; const canCoalesceComments = !isConversation(issue) && opts.allowRunCoalescing !== false; // A resumed receipt already passed admission under this issue lock. // Consume it with its successor even when chat keeps other messages // in separate turns, or finalization will deliver it a second time. const pendingComments = (executionWaitRequestId || canCoalesceComments) && (ownsReconstructionContinuation || !(await getExecutionBlocker(tx as unknown as Db, issue.companyId, issue.id))) ? await tx .select() .from(agentWakeupRequests) .where( and( eq(agentWakeupRequests.companyId, issue.companyId), inArray(agentWakeupRequests.agentId, handoffSource ? [agentId, handoffSource.agentId] : [agentId]), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`, canCoalesceComments || opts.taskRetryRequestId ? undefined : eq(agentWakeupRequests.id, executionWaitRequestId!), ), ) .orderBy(asc(agentWakeupRequests.requestedAt)) : []; const adoptedComments = pendingComments.filter((wake) => { if (wake.id === executionWaitRequestId) return true; if (wake.id === opts.queuedCommentInterruptId || wake.id === opts.queuedCommentRequestId) return true; const deferredPayload = parseObject(wake.payload); const deferredContext = parseObject( deferredPayload[DEFERRED_WAKE_CONTEXT_KEY], ); // Dedicated interaction wakes carry their own source and session // contract. ID-only adoption must not erase that continuation. return ( // Durable chat work must keep its receipt, actor, source, and // session contract through normal promotion and authorization. !wake.idempotencyKey?.startsWith("chat-inbound:") && !isInteractionResolutionWakePayload(deferredPayload) && !hasInteractionContinuationWakeContext(deferredContext) && ["issue_commented", "issue_reopened_via_comment"].includes(String(deferredContext.wakeReason ?? wake.reason)) && queuedCommentIdsFromWakePayload(wake.payload).length > 0 ); }); let adoptedCommentIds = [ ...new Set([ ...adoptedComments.flatMap((wake) => queuedCommentIdsFromWakePayload(wake.payload), ), ...queuedCommentIdsFromRunContext(enrichedContextSnapshot), ]), ]; if (opts.queuedCommentRequestId || (opts.taskRetryRequestId && !failedChatRetry) || handoffSource) { adoptedCommentIds = await undeliveredLegacyUserCommentIds(tx as unknown as Db, agent.companyId, issueId, agentId, adoptedCommentIds); } await notifyChatRunMilestones(tx, { companyId: agent.companyId, issueId: issueId ?? undefined }); const newRun = await tx .insert(heartbeatRuns) .values({ ...(explicitContinuation ? { id: explicitContinuationRunId } : {}), companyId: agent.companyId, agentId, scopeKind: readNonEmptyString(enrichedContextSnapshot.issueId) ? "issue" : "company", issueId: readNonEmptyString(enrichedContextSnapshot.issueId), invocationSource: source, triggerDetail, status: "queued", responsibleUserId: await resolveQueuedResponsibleUserId(), wakeupRequestId: wakeupRequest.id, retryOfRunId: failedChatRetry ? durableRequest!.failedRunRetry!.failedRunId : opts.failedRunId ?? automaticParentRunId, contextSnapshot: adoptedComments.length ? withQueuedCommentIdsInRunContext( enrichedContextSnapshot, adoptedCommentIds, adoptedComments.map((wake) => wake.id), ) : enrichedContextSnapshot, sessionIdBefore: explicitContinuation ? null : sessionBefore, continuationAttempt, ...(reconciledSourceRunId ? { retryOfRunId: reconciledSourceRunId } : {}), ...(reconciledRestoreRetryCount !== null ? { scheduledRetryAttempt: reconciledRestoreRetryCount, scheduledRetryReason: "transient_failure", } : {}), }) .returning() .then((rows) => rows[0]); if (ownsReconstructionContinuation) { await tx.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ continuationDelivery: "delivered", continuationRunId: newRun.id, })}::jsonb` }).where(and(eq(issueRecoveryActions.companyId, issue.companyId), eq(issueRecoveryActions.id, String(enrichedContextSnapshot.recoveryActionId)), sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`)); } await tx .update(agentWakeupRequests) .set({ runId: newRun.id, updatedAt: new Date(), }) .where(eq(agentWakeupRequests.id, wakeupRequest.id)); if (failedChatRetry && opts.taskRetryRequestId) { await tx.update(agentWakeupRequests).set({ status: "coalesced", runId: newRun.id, finishedAt: new Date(), updatedAt: new Date(), }).where(and(eq(agentWakeupRequests.companyId, issue.companyId), eq(agentWakeupRequests.id, opts.taskRetryRequestId), eq(agentWakeupRequests.status, "deferred_issue_execution"))); } if (adoptedComments.length) { await tx .update(agentWakeupRequests) .set({ status: "coalesced", runId: newRun.id, finishedAt: new Date(), updatedAt: new Date(), }) .where( inArray( agentWakeupRequests.id, adoptedComments.map((wake) => wake.id), ), ); await tx .update(agentWakeupRequests) .set({ payload: withQueuedCommentIdsInWakePayload(payload, adoptedCommentIds), }) .where(eq(agentWakeupRequests.id, wakeupRequest.id)); } // executionRunId is NOT stamped here (enqueueWakeup queues the run but // doesn't start it). It will be stamped in claimQueuedRun() once the run // transitions to "running" — Fix A (lazy locking). return { kind: "queued" as const, run: newRun }; }) .catch((error) => { if (isExternalChatWaitAuthorizationContention(error)) return { kind: "deferred" as const }; throw error; }); // Telemetry for the cancelled runs is best-effort background work. // Fire it here and never await it: none of the lifecycle work below, // nor this function's return, depends on it, so a slow telemetry // lookup must not delay them. for (const cancelledRun of cancelledRunsToEmit) { void emitAgentTaskRun(db, cancelledRun); } if (outcome.kind === "durable") { return outcome.receipt.runId ? getRun(outcome.receipt.runId) : null; } if (outcome.kind === "deferred" || outcome.kind === "skipped") { return null; } if (outcome.kind === "coalesced") { await startNextQueuedRunForAgent(agent.id); return outcome.run; } if (outcome.kind === "replayed") { if (outcome.run.status === "queued") await startNextQueuedRunForAgent(agent.id); return outcome.run; } const newRun = outcome.run; publishLiveEvent({ companyId: newRun.companyId, type: "heartbeat.run.queued", payload: { runId: newRun.id, agentId: newRun.agentId, invocationSource: newRun.invocationSource, triggerDetail: newRun.triggerDetail, wakeupRequestId: newRun.wakeupRequestId, }, }); await startNextQueuedRunForAgent(agent.id); return newRun; } if (durableRequest) throw new Error("chat_inbound_wakeup_binding_denied"); const activeRuns = await db .select() .from(heartbeatRuns) .where( and( eq(heartbeatRuns.agentId, agentId), inArray(heartbeatRuns.status, [ ...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES, ]), ), ) .orderBy(desc(heartbeatRuns.createdAt)); const sameScopeQueuedRun = activeRuns.find( (candidate) => candidate.status === "queued" && isSameTaskScope(runTaskKey(candidate), taskKey), ); const sameScopeScheduledRetryRun = activeRuns.find( (candidate) => candidate.status === "scheduled_retry" && isSameTaskScope(runTaskKey(candidate), taskKey), ); const sameScopeRunningRun = activeRuns.find( (candidate) => candidate.status === "running" && isSameTaskScope(runTaskKey(candidate), taskKey), ); const shouldQueueFollowupForRunningWake = Boolean(sameScopeRunningRun) && !sameScopeQueuedRun && shouldQueueFollowupForRunningIssueWake({ contextSnapshot: enrichedContextSnapshot, wakeCommentId, }); // Unscoped manual wakes need their own receipt and execution identity too. const rawCoalescedTarget = opts.allowRunCoalescing === false || opts.manualUserWake ? null : (sameScopeQueuedRun ?? sameScopeScheduledRetryRun ?? (shouldQueueFollowupForRunningWake ? null : (sameScopeRunningRun ?? null))); // A reviewed chat run attests one conversation, so never fold another // conversation's reply into it; that reply queues its own run instead. const coalescedTargetRun = rawCoalescedTarget && wakeChatConversationsConflict(rawCoalescedTarget.contextSnapshot, enrichedContextSnapshot) ? null : filterZombieCoalesceTarget(rawCoalescedTarget, liveRunExecutions); if (coalescedTargetRun) { const mergedContextSnapshot = mergeCoalescedContextSnapshot( coalescedTargetRun.contextSnapshot, enrichedContextSnapshot, { preserveExistingInteractionContinuation: coalescedTargetRun.status === "queued" || coalescedTargetRun.status === "scheduled_retry", }, ); const mergedRun = await db .update(heartbeatRuns) .set({ contextSnapshot: mergedContextSnapshot, updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, coalescedTargetRun.id)) .returning() .then((rows) => rows[0] ?? coalescedTargetRun); await db.insert(agentWakeupRequests).values({ ...durableReceiptFields, companyId: agent.companyId, agentId, source, triggerDetail, reason, payload, status: "coalesced", coalescedCount: 1, requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, runId: mergedRun.id, finishedAt: new Date(), }); return mergedRun; } const queueOutcome = await db.transaction(async (tx) => { await tx.execute( sql`select id from agents where id = ${agentId} and company_id = ${agent.companyId} for update`, ); const dailyCapBlock = await getHeartbeatDailyCapBlock( agent, policy, {}, tx, ); if (dailyCapBlock) { const now = new Date(); await tx.insert(agentWakeupRequests).values({ ...durableReceiptFields, companyId: agent.companyId, agentId, source, triggerDetail, reason: dailyCapBlock.reason, payload: { ...(payload ?? {}), heartbeatSkip: { reason: "Per-agent heartbeat daily cap reached before adapter invocation.", observed: dailyCapBlock.observed, limit: dailyCapBlock.limit, }, }, status: "skipped", requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, finishedAt: now, }); if (source === "timer") { await tx .update(agents) .set({ lastHeartbeatAt: now, updatedAt: now, }) .where(eq(agents.id, agentId)); } return { kind: "skipped" as const }; } const wakeupRequest = await tx .insert(agentWakeupRequests) .values({ ...durableReceiptFields, companyId: agent.companyId, agentId, source, triggerDetail, reason, payload, status: "queued", requestedByActorType: opts.requestedByActorType ?? null, requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, }) .returning() .then((rows) => rows[0]); const newRun = await tx .insert(heartbeatRuns) .values({ companyId: agent.companyId, agentId, scopeKind: readNonEmptyString(enrichedContextSnapshot.issueId) ? "issue" : "company", issueId: readNonEmptyString(enrichedContextSnapshot.issueId), invocationSource: source, triggerDetail, status: "queued", responsibleUserId: await resolveQueuedResponsibleUserId(), wakeupRequestId: wakeupRequest.id, contextSnapshot: enrichedContextSnapshot, sessionIdBefore: sessionBefore, continuationAttempt, }) .returning() .then((rows) => rows[0]); await tx .update(agentWakeupRequests) .set({ runId: newRun.id, updatedAt: new Date(), }) .where(eq(agentWakeupRequests.id, wakeupRequest.id)); return { kind: "queued" as const, run: newRun }; }); if (queueOutcome.kind === "skipped") return null; const newRun = queueOutcome.run; publishLiveEvent({ companyId: newRun.companyId, type: "heartbeat.run.queued", payload: { runId: newRun.id, agentId: newRun.agentId, invocationSource: newRun.invocationSource, triggerDetail: newRun.triggerDetail, wakeupRequestId: newRun.wakeupRequestId, }, }); await startNextQueuedRunForAgent(agent.id); return newRun; } /** * Native status commitment deliberately persists dependency/parent wake * intents in the same transaction as the authoritative status projection. * Those rows are not runnable until the heartbeat scheduler has applied its * normal policy, workspace, concurrency, and responsible-user checks. Bridge * the durable intent into that scheduler here instead of treating a bare * `agent_wakeup_requests` row as if it were already a queued heartbeat run. * * The intent is claimed before dispatch. A deterministic dispatcher actor id * lets a later sweep recover the narrow process-crash window after the real * wake was inserted but before the intent was linked to it. Dispatch failure * only requeues the intent; it never changes the provider run outcome. */ async function dispatchPendingNativeStatusWakeups( input: { companyId?: string; limit?: number; staleClaimMs?: number; } = {}, ) { const now = new Date(); const staleClaimMs = Math.max(1_000, input.staleClaimMs ?? 60_000); const candidates = await db .select() .from(agentWakeupRequests) .where( and( input.companyId ? eq(agentWakeupRequests.companyId, input.companyId) : undefined, eq(agentWakeupRequests.requestedByActorType, "system"), eq(agentWakeupRequests.requestedByActorId, "native-status-committer"), inArray(agentWakeupRequests.status, ["queued", "claimed"]), isNull(agentWakeupRequests.runId), ), ) .orderBy(asc(agentWakeupRequests.requestedAt)) .limit(Math.max(1, Math.min(input.limit ?? 100, 500))); let dispatched = 0; let recovered = 0; let deferred = 0; const deliveredByIssueScope = new Map< string, { runId: string | null; status: string } >(); for (const candidate of candidates) { const payload = parseObject(candidate.payload); const wakeContext = parseObject(payload._paperclipWakeContext); const issueId = readNonEmptyString(payload.issueId) ?? readNonEmptyString(payload.taskId) ?? readNonEmptyString(wakeContext.issueId) ?? null; const dispatchActorId = `native-status-wake-dispatch:${candidate.id}`; const existingDispatch = await db .select() .from(agentWakeupRequests) .where( and( eq(agentWakeupRequests.companyId, candidate.companyId), eq(agentWakeupRequests.requestedByActorType, "system"), eq(agentWakeupRequests.requestedByActorId, dispatchActorId), ), ) .orderBy(desc(agentWakeupRequests.requestedAt)) .limit(1) .then((rows) => rows[0] ?? null); if (existingDispatch) { const recoveredStatus = existingDispatch.runId ? "coalesced" : existingDispatch.status === "deferred_issue_execution" ? "coalesced" : existingDispatch.status; await db.transaction(async tx => { await notifyChatRunMilestones(tx, { companyId: candidate.companyId, issueId: issueId ?? undefined }); await tx .update(agentWakeupRequests) .set({ status: recoveredStatus, runId: existingDispatch.runId, finishedAt: (existingDispatch.runId ?? existingDispatch.finishedAt) ? (existingDispatch.finishedAt ?? now) : null, error: existingDispatch.error, updatedAt: now, }) .where( and( eq(agentWakeupRequests.id, candidate.id), isNull(agentWakeupRequests.runId), ), ); }); recovered += 1; continue; } if ( candidate.status === "claimed" && candidate.claimedAt && now.getTime() - candidate.claimedAt.getTime() < staleClaimMs ) { deferred += 1; continue; } const claimed = await db .update(agentWakeupRequests) .set({ status: "claimed", claimedAt: now, error: null, updatedAt: now, }) .where( and( eq(agentWakeupRequests.id, candidate.id), isNull(agentWakeupRequests.runId), candidate.status === "claimed" ? eq(agentWakeupRequests.status, "claimed") : eq(agentWakeupRequests.status, "queued"), ), ) .returning({ id: agentWakeupRequests.id }); if (claimed.length === 0) continue; const scopeKey = issueId ? `${candidate.companyId}:${candidate.agentId}:${issueId}:${readNativeReviewAssignmentContext(wakeContext)?.nativeReviewInteractionId ?? ""}` : null; const priorDelivery = scopeKey ? deliveredByIssueScope.get(scopeKey) : null; if (priorDelivery) { await db.transaction(async tx => { await notifyChatRunMilestones(tx, { companyId: candidate.companyId, issueId: issueId ?? undefined }); await tx .update(agentWakeupRequests) .set({ status: "coalesced", runId: priorDelivery.runId, finishedAt: new Date(), error: null, updatedAt: new Date(), }) .where(eq(agentWakeupRequests.id, candidate.id)); }); recovered += 1; continue; } let completedOnboardingGuard: WakeupOptions["issueStateGuard"]; if (issueId) { const targetIssue = await db .select({ status: issues.status, statusVersion: issues.statusVersion, assigneeAgentId: issues.assigneeAgentId, }) .from(issues) .where( and( eq(issues.id, issueId), eq(issues.companyId, candidate.companyId), ), ) .limit(1) .then((rows) => rows[0] ?? null); const nativeReview = candidate.reason === "native_completion_review" ? await getNativeReviewAssignment(db, { companyId: candidate.companyId, issueId, agentId: candidate.agentId, contextSnapshot: wakeContext, }) : null; const onboardingResultReport = targetIssue?.status === "done" && await isCompletedOnboardingHandoffWake(db, { companyId: candidate.companyId, issueId, agentId: candidate.agentId, reason: candidate.reason, contextSnapshot: wakeContext, }); if (onboardingResultReport) completedOnboardingGuard = { assigneeAgentId: candidate.agentId, statuses: ["done"], statusVersion: targetIssue!.statusVersion }; if ( !targetIssue || (["done", "cancelled"].includes(targetIssue.status) && !onboardingResultReport) || (targetIssue.assigneeAgentId !== candidate.agentId && !nativeReview) || (candidate.reason === "native_completion_review" && !nativeReview) ) { await db .update(agentWakeupRequests) .set({ status: "skipped", finishedAt: new Date(), error: !targetIssue ? "Native status wake target no longer exists" : ["done", "cancelled"].includes(targetIssue.status) ? `Native status wake target is already ${targetIssue.status}` : "Native status wake target has a different assignee", updatedAt: new Date(), }) .where(eq(agentWakeupRequests.id, candidate.id)); recovered += 1; continue; } } try { const wakeRun = await enqueueWakeup(candidate.agentId, { allowRunCoalescing: false, source: candidate.source as WakeupOptions["source"], triggerDetail: (candidate.triggerDetail ?? "system") as WakeupOptions["triggerDetail"], reason: candidate.reason, payload, idempotencyKey: candidate.idempotencyKey, requestedByActorType: "system", requestedByActorId: dispatchActorId, ...(completedOnboardingGuard ? { issueStateGuard: completedOnboardingGuard } : {}), contextSnapshot: { ...wakeContext, ...(issueId ? { issueId, taskId: issueId } : {}), wakeReason: candidate.reason, source: "native_status_decision", statusDecisionSource: "native_status_decision", nativeStatusWakeIntentId: candidate.id, }, }); const delivered = await db .select() .from(agentWakeupRequests) .where( and( eq(agentWakeupRequests.companyId, candidate.companyId), eq(agentWakeupRequests.requestedByActorType, "system"), eq(agentWakeupRequests.requestedByActorId, dispatchActorId), ), ) .orderBy(desc(agentWakeupRequests.requestedAt)) .limit(1) .then((rows) => rows[0] ?? null); // The committer intent is the durable outbox entry. When admission is // blocked, enqueueWakeup creates a separate deferred dispatch receipt; // leave the original intent coalesced so the release drain cannot // promote both rows (the original has no nativeStatusWakeIntentId // provenance and would otherwise run once before the dispatch receipt). const deliveredStatus = delivered?.status ?? "queued"; await db.transaction(async tx => { await notifyChatRunMilestones(tx, { companyId: candidate.companyId, issueId: issueId ?? undefined }); await tx .update(agentWakeupRequests) .set({ status: wakeRun || deliveredStatus === "deferred_issue_execution" ? "coalesced" : deliveredStatus, runId: wakeRun?.id ?? delivered?.runId ?? null, finishedAt: wakeRun || delivered?.finishedAt ? (delivered?.finishedAt ?? new Date()) : null, error: delivered?.error ?? null, updatedAt: new Date(), }) .where(eq(agentWakeupRequests.id, candidate.id)); }); if (scopeKey) { deliveredByIssueScope.set(scopeKey, { runId: wakeRun?.id ?? delivered?.runId ?? null, status: wakeRun ? "coalesced" : (delivered?.status ?? "queued"), }); } if (wakeRun) dispatched += 1; else deferred += 1; } catch (error) { await db .update(agentWakeupRequests) .set({ status: "queued", claimedAt: null, error: error instanceof Error ? error.message.slice(0, 1_000) : String(error).slice(0, 1_000), updatedAt: new Date(), }) .where( and( eq(agentWakeupRequests.id, candidate.id), eq(agentWakeupRequests.status, "claimed"), isNull(agentWakeupRequests.runId), ), ); logger.warn( { err: error, wakeupRequestId: candidate.id, agentId: candidate.agentId, }, "failed to dispatch persisted native status wake intent", ); deferred += 1; } } return { scanned: candidates.length, dispatched, recovered, deferred }; } return { enqueueWakeup, dispatchPendingNativeStatusWakeups, startNextQueuedRunForAgent, countRunningRunsForAgent, releaseRunClaimedJustBeforeSuppression, claimQueuedRun, withChatControlRecoveryGate, trackWakeup, resumeQueuedRuns, parseHeartbeatPolicy, claimDueTimerHeartbeat, }; }