From a836109fb0ba5387fead3f7426287e4b3c8ba767 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:17:28 -0500 Subject: [PATCH] Restore complete parent sources in local integration resolution Reconstruct conflicted files from immutable parents and apply only line-bounded additive resolutions. Keep this unpublished integration correction visible in history. Co-Authored-By: Paperclip --- .../modules/wake-queue/adapters/postgres.ts | 1329 ++ server/src/services/budgets.ts | 1157 ++ server/src/services/companies.ts | 561 + .../services/delivery-work-notifications.ts | 132 + .../execution-control-notifications.ts | 31 + server/src/services/heartbeat/queue.ts | 5083 ++++++ server/src/services/heartbeat/retries.ts | 1933 +++ server/src/services/instance-settings.ts | 670 + .../issue-review-state-notifications.ts | 14 + server/src/services/issues.ts | 14119 ++++++++++++++++ 10 files changed, 25029 insertions(+) diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index 21b8da13658..125b0c86984 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -1 +1,1330 @@ +import { notifyRunDispatch } from "../../../services/run-dispatch-notifications.js"; import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js"; +import { notifyExternalConversationSettlement } from "../../../services/external-conversation-settlement-notifications.js"; +import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js"; +import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js"; +import { notifyChatInboundWakeNotices } from "../../../services/chat-inbound-wake-notice-notifications.js"; +import { notifyChatRunMilestones } from "../../../services/chat-run-milestone-notifications.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../../../services/delivery-work-notifications.js"; +import { isAcknowledgedNativeReassignmentStop, isAcknowledgedNativeStop } from "../../../services/acknowledged-native-stop.js"; +import { readRunCancellation } from "../../../services/run-cancellation.js"; +import { isCompletedOnboardingHandoffWake } from "../../../services/chat-completion-delivery.js"; +import { instanceSettingsService } from "../../../services/instance-settings.js"; +import { currentConversationCommentCondition } from "../../../services/agent-conversations.js"; +import { getExecutionBlocker, hasPendingRunCleanup } from "../../../services/execution-blocker.js"; +import { and, asc, eq, inArray, isNull, notInArray, or, sql } from "drizzle-orm"; +import { readReleaseRun, type ReleaseRun } from "./release-run-read.js"; +import type { Db } from "@paperclipai/db"; +import { extractIssueReferenceIdentifiers, isUuidLike } from "@paperclipai/shared"; +import { + activityLog, + agentWakeupRequests, + agents, + chatActions, + heartbeatRuns, + issueComments, + issueRecoveryActions, + issueRelations, + issues, + nativeRunFinalizations, +} from "@paperclipai/db"; +import { hasConversationContinuationPolicy } from "../../../services/conversation-continuation.js"; +import { legacyExecutionNeedsReconciliationWithEvidence } from "../../../services/legacy-execution-recovery.js"; +import { + authorizeFailedChatRunRetryWake, + FailedChatRunRetryAuthorizationError, +} from "../../../services/durable-chat-wakeup.js"; +import { isRetiredExternalChatQuestionSource } from "../../../services/question-response-delivery.js"; +import { HttpError } from "../../../errors.js"; +import { evaluateAgentInvokabilityFromDb } from "../../../services/agent-invokability.js"; +import { issueTreeControlService, isVerifiedIssueTreeControlInteractionWake } from "../../../services/issue-tree-control.js"; +import { isAutomaticRecoverySuppressedByPauseHold } from "../../../services/recovery/pause-hold-guard.js"; +import { classifyContinuationFailure } from "../../../services/recovery/service.js"; +import { issueService } from "../../../services/issues.js"; +import { issueRecoveryActionService } from "../../../services/issue-recovery-actions.js"; +import { readContinuationAttempt } from "../../../services/recovery/run-liveness-continuations.js"; +import { withRecoveryContext } from "../../../services/recovery/status-only-context.js"; +import { parseIssueExecutionState } from "../../../services/issue-execution-policy.js"; +import { + queuedCommentIdsFromWakePayload, + withQueuedCommentIdsInWakePayload, +} from "../../../services/issue-queued-comment-queue.js"; +import { CHAT_CONVERSATION_ID_KEY, extractWakeCommentIds } from "../../run-dispatch/index.js"; +import { hasInteractionContinuationWakeContext } from "../domain/context.js"; +import { decidePreDrain, type PreDrainFacts } from "../domain/policy.js"; +import { + EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON, + isConfigurationIncompleteFailedRun, + isWorkspaceValidationFailedRun, + parseObject, + readNonEmptyString, +} from "../domain/values.js"; +import { requireTransactionScopeTx, TransactionScope } from "../application/ports.js"; +import type { + DeferredWakeCandidate, + InvokableAgentSnapshot, + IssueLockWriter, + IssueSnapshot, + LockedIssueExecution, + ReleaseTransactionResult, + RunSnapshot, + WakeAdmissionReader, + WakeAdmissionWriter, + WakeQueueHost, + WakeQueueTransaction, +} from "../application/ports.js"; +import type { RunSummary } from "../application/types.js"; + +const DEFERRED_WAKE_STATUS = "deferred_issue_execution"; +const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext"; +const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const; + +type HeartbeatRunRow = typeof heartbeatRuns.$inferSelect; +type IssueRow = typeof issues.$inferSelect; + +function normalizeAgentNameKey(value: string | null | undefined): string | null { + if (typeof value !== "string") return null; + const normalized = value.trim().toLowerCase(); + return normalized.length > 0 ? normalized : null; +} + +function toRequestedByActorType(value: string | null): "user" | "agent" | "system" | null { + // The database column is free text; map any value outside the union to + // null instead of widening the type back to string. + return value === "user" || value === "agent" || value === "system" ? value : null; +} + +function toRunSnapshot(row: ReleaseRun): RunSnapshot { + const configurationIncompletePayload = parseObject(parseObject(row.resultJson).configurationIncomplete); + return { + id: row.id, + companyId: row.companyId, + agentId: row.agentId, + status: row.status, + runtimeMode: row.runtimeMode, + conversationContinuation: row.runtimeMode === "legacy" && hasConversationContinuationPolicy(row.resultJson), + errorCode: row.errorCode, + responsibleUserId: row.responsibleUserId, + contextSnapshot: parseObject(row.contextSnapshot), + configurationIncompletePayload: Object.keys(configurationIncompletePayload).length > 0 ? configurationIncompletePayload : null, + operatorStopped: isOperatorStoppedRunRow(row), + }; +} + +const OPERATOR_STOP_ERROR_CODES = new Set(["operator_interrupted", "agent_paused"]); + +/** Mirrors recovery's `isOperatorCancelledRun`, plus agent pause, for the release-time recovery tail. */ +export function isOperatorStoppedRunRow(row: Pick): boolean { + if (row.status !== "cancelled") return false; + const cancellation = readRunCancellation(row.resultJson); + if (cancellation?.source === "queued_message") return false; + if (cancellation?.source === "operator" && cancellation.expected) return true; + if (row.errorCode && OPERATOR_STOP_ERROR_CODES.has(row.errorCode)) return true; + const actorType = parseObject(row.resultJson).cancelledByActorType; + return actorType === "user" || actorType === "board"; +} + +function toIssueSnapshot(row: IssueRow): IssueSnapshot { + return { + conversationAgentId: row.conversationAgentId, + conversationUserId: row.conversationUserId, + conversationState: row.conversationState, + id: row.id, + companyId: row.companyId, + identifier: row.identifier ?? "", + status: row.status, + assigneeAgentId: row.assigneeAgentId, + assigneeUserId: row.assigneeUserId, + hiddenAt: row.hiddenAt, + originKind: row.originKind, + monitorNextCheckAt: row.monitorNextCheckAt, + executionState: (row.executionState as Record | null) ?? null, + responsibleUserId: row.responsibleUserId, + parentId: row.parentId, + originId: row.originId, + originRunId: row.originRunId, + }; +} + +function toRunSummary(row: HeartbeatRunRow): RunSummary { + return { + id: row.id, + companyId: row.companyId, + agentId: row.agentId, + invocationSource: row.invocationSource, + triggerDetail: row.triggerDetail, + wakeupRequestId: row.wakeupRequestId, + }; +} + +function toDeferredWakeCandidate(row: typeof agentWakeupRequests.$inferSelect): DeferredWakeCandidate { + const payload = parseObject(row.payload); + const queuedCommentIds = queuedCommentIdsFromWakePayload(payload); + const deferredContextSeed = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]); + const deferredCommentIds = extractWakeCommentIds(deferredContextSeed); + const wakeReason = readNonEmptyString(deferredContextSeed.wakeReason); + const queuedReason = wakeReason ?? readNonEmptyString(row.reason); + const queuedWakeIsCommentOnly = + !queuedReason || + queuedReason === "issue_commented" || + queuedReason === "issue_reopened_via_comment" || + queuedReason === "issue_comment_mentioned"; + const preservesIndependentContinuation = + hasInteractionContinuationWakeContext(deferredContextSeed) || + deferredContextSeed.resumeIntent === true || + !queuedWakeIsCommentOnly; + + return { + id: row.id, + companyId: row.companyId, + agentId: row.agentId, + reason: row.reason, + source: row.source, + triggerDetail: row.triggerDetail, + requestedByActorType: toRequestedByActorType(row.requestedByActorType), + requestedByActorId: row.requestedByActorId, + payload, + queuedCommentIds, + preservesIndependentContinuation, + deferredContextSeed, + deferredCommentIds, + wakeReason, + }; +} + +export type WakeQueuePostgresAdapterDeps = { + resolveResponsibleUserId: WakeQueueHost["resolveResponsibleUserId"]; + getRoutineEnv: WakeQueueHost["getRoutineEnv"]; + resolveSessionBeforeForWakeup: WakeQueueHost["resolveSessionBeforeForWakeup"]; +}; + +function buildHost(_tx: Db, deps: WakeQueuePostgresAdapterDeps): WakeQueueHost { + return { + resolveResponsibleUserId: deps.resolveResponsibleUserId, + getRoutineEnv: deps.getRoutineEnv, + resolveSessionBeforeForWakeup: deps.resolveSessionBeforeForWakeup, + }; +} + +function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, run: ReleaseRun): WakeQueueTransaction { + const treeControlSvc = issueTreeControlService(tx); + const issuesSvc = issueService(tx); + const interruptQueueId = run.runtimeMode !== "native" && run.status === "cancelled" + ? readNonEmptyString(run.resultJson?.queuedCommentInterruptQueueId) + : null; + + return { + async findInvokableAgent({ companyId, agentId }): Promise { + const agent = await tx + .select() + .from(agents) + .where(and(eq(agents.id, agentId), eq(agents.companyId, companyId))) + .then((rows) => rows[0] ?? null); + if (!agent) return null; + const invokability = await evaluateAgentInvokabilityFromDb(tx, agent); + return { id: agent.id, companyId: agent.companyId, name: agent.name, invokable: invokability.invokable, + paused: !invokability.invokable && invokability.reason === "paused" }; + }, + + async findNextDeferredWake({ companyId, issueId, excludedWakeIds }) { + while (true) { + const row = await tx + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + excludedWakeIds?.length ? notInArray(agentWakeupRequests.id, excludedWakeIds) : undefined, + sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`, + interruptQueueId ? eq(agentWakeupRequests.id, interruptQueueId) : undefined, + interruptQueueId ? eq(agentWakeupRequests.agentId, run.agentId) : undefined, + ), + ) + .orderBy(asc(agentWakeupRequests.requestedAt)) + .limit(1) + .then((rows) => rows[0] ?? null); + if (!row) return null; + const candidate = toDeferredWakeCandidate(row); + try { + const authorizedFailedChatRetry = await authorizeFailedChatRunRetryWake( + db, + tx, + { + phase: "promotion", + wakeupRequestId: row.id, + companyId, + agentId: row.agentId, + issueId, + contextSnapshot: candidate.deferredContextSeed, + }, + ); + return { ...candidate, authorizedFailedChatRetry }; + } catch (error) { + // Retire only a proven denial. Transient failures roll back this + // transaction instead of discarding otherwise authorized work. + if ( + !(error instanceof FailedChatRunRetryAuthorizationError) && + !( + error instanceof HttpError && + error.status >= 400 && + error.status < 500 + ) + ) { + throw error; + } + const now = new Date(); + await notifyChatInboundWakeNotices(tx, { companyId, issueId }); + await tx + .update(agentWakeupRequests) + .set({ + status: "cancelled", + finishedAt: now, + error: + "The exact failed chat request is no longer authorized. Send a new request in the current connected conversation.", + updatedAt: now, + }) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.id, row.id), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + ), + ); + } + } + }, + + async getQueuedCommentLiveness({ companyId, issueId, wakeAgentId, finishingRunId, finishingRunAgentId, queuedCommentIds }) { + // Malformed ids (for example an 8-char short form written into a wake + // payload) can never match a comment row, but Postgres rejects the whole + // `inArray` statement on the first invalid uuid and blocks every queued + // wake for the issue. Skip them up front so one bad id cannot take down + // the batch; the liveness filters below already treat them as not live. + const queryableCommentIds = queuedCommentIds + .map((commentId) => commentId.trim()) + .filter((commentId) => isUuidLike(commentId)); + const rows = queryableCommentIds.length === 0 ? [] : await tx + .select({ id: issueComments.id, deletedAt: issueComments.deletedAt, createdByRunId: issueComments.createdByRunId }) + .from(issueComments) + .where(and(eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), inArray(issueComments.id, queryableCommentIds), currentConversationCommentCondition())); + const targetsFinishingRunAgent = wakeAgentId === finishingRunAgentId; + const liveNonSelfCommentIds = queryableCommentIds.filter((commentId) => { + const row = rows.find((candidate) => candidate.id === commentId); + return Boolean(row && !row.deletedAt && (!targetsFinishingRunAgent || row.createdByRunId !== finishingRunId)); + }); + const containedSelfAuthoredComment = rows.some( + (row) => targetsFinishingRunAgent && !row.deletedAt && row.createdByRunId === finishingRunId, + ); + return { liveNonSelfCommentIds, containedSelfAuthoredComment }; + }, + + async cancelDeferredWake({ companyId, wakeId, reason, now }) { + await notifyChatInboundWakeNotices(tx, { companyId }); + const rows = await tx + .update(agentWakeupRequests) + .set({ status: "cancelled", finishedAt: now, error: reason, updatedAt: now }) + .where( + and( + eq(agentWakeupRequests.id, wakeId), + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + ), + ) + .returning({ id: agentWakeupRequests.id }); + return rows.length > 0; + }, + + async normalizeDeferredWakeCommentIds({ companyId, wakeId, payload, liveCommentIds, now }) { + const rows = await tx + .update(agentWakeupRequests) + .set({ payload: withQueuedCommentIdsInWakePayload(payload, liveCommentIds), updatedAt: now }) + .where( + and( + eq(agentWakeupRequests.id, wakeId), + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + ), + ) + .returning(); + const row = rows[0]; + return row ? toDeferredWakeCandidate(row) : null; + }, + + async failDeferredWake({ companyId, wakeId, now }) { + await notifyChatInboundWakeNotices(tx, { companyId }); + const rows = await tx + .update(agentWakeupRequests) + .set({ + status: "failed", + finishedAt: now, + error: "Deferred wake could not be promoted: agent is not invokable", + updatedAt: now, + }) + .where( + and( + eq(agentWakeupRequests.id, wakeId), + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + ), + ) + .returning({ id: agentWakeupRequests.id }); + return rows.length > 0; + }, + + async getPauseHoldFacts({ companyId, issueId, wakeAgentId, deferredContextSeed, requestedByActorType, requestedByActorId }) { + const activePauseHold = await treeControlSvc.getActivePauseHoldGate(companyId, issueId); + if (!activePauseHold) { + return { + activePauseHold: false, + treeHoldInteractionWake: false, + holdId: null, + rootIssueId: null, + mode: null, + reason: null, + releasePolicy: null, + }; + } + const treeHoldInteractionWake = await isVerifiedIssueTreeControlInteractionWake(tx, { + companyId, + issueId, + agentId: wakeAgentId, + contextSnapshot: deferredContextSeed, + requestedByActorType, + requestedByActorId, + }); + return { + activePauseHold: true, + treeHoldInteractionWake, + holdId: activePauseHold.holdId, + rootIssueId: activePauseHold.rootIssueId, + mode: activePauseHold.mode, + reason: activePauseHold.reason, + releasePolicy: activePauseHold.releasePolicy, + }; + }, + + async getCommentSelfAuthorship({ companyId, issueId, finishingRunId, commentIds }) { + const rows = await tx + .select({ createdByRunId: issueComments.createdByRunId }) + .from(issueComments) + .where(and(eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), inArray(issueComments.id, commentIds))); + return { allSelfAuthored: rows.length > 0 && rows.every((row) => row.createdByRunId === finishingRunId) }; + }, + + async isCompletedDelegationMention({ companyId, issueId, finishingRunId, wakeAgentId, commentIds }) { + if ( + run.companyId !== companyId || run.id !== finishingRunId || + readNonEmptyString(parseObject(run.contextSnapshot).issueId) !== issueId + ) return false; + const uniqueCommentIds = [...new Set(commentIds)]; + if (uniqueCommentIds.length === 0) return false; + const parent = await tx.select({ status: issues.status, assigneeAgentId: issues.assigneeAgentId, identifier: issues.identifier }) + .from(issues).where(and(eq(issues.companyId, companyId), eq(issues.id, issueId))) + .then((rows) => rows[0]); + if (parent?.status !== "done" || parent.assigneeAgentId !== run.agentId) return false; + + const comments = await tx.select({ body: issueComments.body, createdByRunId: issueComments.createdByRunId, + authorAgentId: issueComments.authorAgentId }) + .from(issueComments).where(and( + eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), + inArray(issueComments.id, uniqueCommentIds), isNull(issueComments.deletedAt), currentConversationCommentCondition(), + )); + if (comments.length !== uniqueCommentIds.length || comments.some((comment) => + comment.createdByRunId !== run.id || (comment.authorAgentId !== null && comment.authorAgentId !== run.agentId) + )) return false; + const referencesByComment = comments.map((comment) => extractIssueReferenceIdentifiers(comment.body) + .filter((identifier) => identifier !== parent.identifier)); + // A parent link is context; every other reference must identify the + // single completed child. Never discard unrelated follow-up work. + if (referencesByComment.some((references) => references.length !== 1)) return false; + const identifiers = [...new Set(referencesByComment.flat())]; + const children = await tx.select({ identifier: issues.identifier, status: issues.status }) + .from(issues).where(and( + eq(issues.companyId, companyId), eq(issues.parentId, issueId), + eq(issues.assigneeAgentId, wakeAgentId), inArray(issues.identifier, identifiers), + )); + return referencesByComment.every((references) => { + const referencedChildren = children.filter((child) => child.identifier !== null && references.includes(child.identifier)); + return referencedChildren.length === 1 && referencedChildren[0].status === "done"; + }); + }, + + async isCompletedOnboardingHandoffWake(input) { + return isCompletedOnboardingHandoffWake(tx, input); + }, + + async reopenIssue({ companyId, issueId }) { + const updated = await issuesSvc.updateForCompany(issueId, companyId, { status: "todo", executionState: null }, tx); + return updated ? toIssueSnapshot(updated as unknown as IssueRow) : null; + }, + + async claimDeferredWakeForPromotion({ companyId, wakeId, now }) { + const claimed = await tx + .update(agentWakeupRequests) + .set({ + status: "queued", + reason: "issue_execution_promoted", + claimedAt: null, + finishedAt: null, + error: null, + updatedAt: now, + }) + .where( + and( + eq(agentWakeupRequests.id, wakeId), + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + ), + ) + .returning({ id: agentWakeupRequests.id }); + return claimed.length > 0; + }, + + async finalizePromotedWake(input) { + await notifyChatRunMilestones(tx, { companyId: input.companyId, issueId: input.issue.id }); + const newRun = await tx + .insert(heartbeatRuns) + .values({ + companyId: input.deferredAgent.companyId, + agentId: input.deferredAgent.id, + invocationSource: input.source, + triggerDetail: input.triggerDetail, + status: "queued", + wakeupRequestId: input.wakeId, + retryOfRunId: input.authorizedFailedChatRetry + ? readNonEmptyString(input.contextSnapshot.retryOfRunId) + : null, + contextSnapshot: input.contextSnapshot, + responsibleUserId: input.responsibleUserId, + sessionIdBefore: input.sessionBefore, + continuationAttempt: readContinuationAttempt(input.contextSnapshot.livenessContinuationAttempt), + }) + .returning() + .then((rows) => rows[0]); + + // `claimDeferredWakeForPromotion` already moved this row off + // `deferred_issue_execution` inside this same transaction, so no + // concurrent claimer can still match that guard; this extra `runId is + // null` guard only protects against writing the link twice. + await tx + .update(agentWakeupRequests) + .set({ runId: newRun.id, updatedAt: input.now }) + .where( + and( + eq(agentWakeupRequests.id, input.wakeId), + eq(agentWakeupRequests.companyId, input.companyId), + isNull(agentWakeupRequests.runId), + ), + ); + + // Promoted mention wakes are issue-scoped, not issue ownership + // transfers. The lock-clearing step earlier in this transaction + // already set `executionRunId` to null for this issue, so the `is + // null` guard only protects against taking the lock twice. + await tx + .update(issues) + .set({ + executionRunId: newRun.id, + executionAgentNameKey: normalizeAgentNameKey(input.deferredAgent.name), + executionLockedAt: input.now, + updatedAt: input.now, + }) + .where( + and( + eq(issues.id, input.issue.id), + eq(issues.companyId, input.companyId), + eq(issues.assigneeAgentId, input.deferredAgent.id), + isNull(issues.executionRunId), + ), + ); + + return toRunSummary(newRun); + }, + + async hasExistingExecutionPath({ companyId, issueId, excludeRunId, agentId }) { + const row = await tx + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.status, [...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES]), + sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`, + sql`${heartbeatRuns.id} <> ${excludeRunId}`, + agentId ? eq(heartbeatRuns.agentId, agentId) : sql`true`, + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + return row !== null; + }, + + async hasExplicitBlockerPath({ companyId, issueId }) { + const row = await tx + .select({ issueId: issueRelations.issueId }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.relatedIssueId, issueId), + eq(issueRelations.type, "blocks"), + eq(issues.companyId, companyId), + notInArray(issues.status, ["done", "cancelled"]), + isNull(issues.hiddenAt), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + return row !== null; + }, + + async isAutomaticRecoverySuppressedByPauseHold({ companyId, issueId }) { + return isAutomaticRecoverySuppressedByPauseHold(tx, companyId, issueId, treeControlSvc); + }, + + async isImmediateRecoverySourceBlocked({ companyId, runId }) { + if (companyId !== run.companyId || runId !== run.id) { + throw new Error( + "wake-queue: recovery source does not match the locked execution", + ); + } + const failedChatRequestOwner = run.wakeupRequestId + ? await tx + .select({ id: chatActions.id }) + .from(chatActions) + .where( + and( + eq(chatActions.id, run.wakeupRequestId), + eq(chatActions.companyId, run.companyId), + inArray(chatActions.kind, ["inbound_wakeup", "failed_run_retry"]), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null) + : null; + return ( + Boolean(failedChatRequestOwner) || + run.errorCode === "chat_failed_run_retry_not_authorized" || + classifyContinuationFailure(run).kind === "non_retryable" + ); + }, + + async queueReviewParticipantRecoveryRun({ companyId, issue, finishingRun, recoveryAgent, sessionBefore, now }) { + await notifyStatusCardIssueChange(tx, companyId); + const executionState = parseIssueExecutionState(issue.executionState); + const wakeupRequest = await tx + .insert(agentWakeupRequests) + .values({ + companyId, + agentId: recoveryAgent.id, + source: "automation", + triggerDetail: "system", + reason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON, + payload: withRecoveryContext( + { + issueId: issue.id, + retryOfRunId: finishingRun.id, + retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON, + currentStageId: executionState?.currentStageId ?? null, + currentStageType: executionState?.currentStageType ?? null, + }, + "normal_model", + ), + status: "queued", + requestedByActorType: "system", + requestedByActorId: null, + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + await notifyRunDispatch(tx, { companyId }); + const queuedRun = await tx + .insert(heartbeatRuns) + .values({ + companyId, + agentId: recoveryAgent.id, + invocationSource: "automation", + triggerDetail: "system", + status: "queued", + wakeupRequestId: wakeupRequest.id, + contextSnapshot: withRecoveryContext( + { + issueId: issue.id, + taskId: issue.id, + wakeReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON, + retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON, + source: "issue.execution_review_recovery", + retryOfRunId: finishingRun.id, + currentStageId: executionState?.currentStageId ?? null, + currentStageType: executionState?.currentStageType ?? null, + reviewRecoveryInstruction: + "The previous reviewer run ended while this execution-review stage was still pending. Submit the review decision now, or mark the issue blocked with the exact unblock action.", + }, + "normal_model", + ), + sessionIdBefore: sessionBefore, + retryOfRunId: finishingRun.id, + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + await tx + .update(agentWakeupRequests) + .set({ runId: queuedRun.id, updatedAt: now }) + .where(and(eq(agentWakeupRequests.id, wakeupRequest.id), eq(agentWakeupRequests.companyId, companyId))); + + await tx + .update(issues) + .set({ + executionRunId: queuedRun.id, + executionAgentNameKey: normalizeAgentNameKey(recoveryAgent.name), + executionLockedAt: now, + updatedAt: now, + }) + .where(and(eq(issues.id, issue.id), eq(issues.companyId, companyId))); + + return toRunSummary(queuedRun); + }, + + async queueImmediateRecoveryRun({ + companyId, + issue, + finishingRun, + recoveryAgent, + reason, + contextSnapshot, + responsibleUserId, + sessionBefore, + now, + }) { + await notifyChatRunMilestones(tx, { companyId, issueId: issue.id }); + const wakeupRequest = await tx + .insert(agentWakeupRequests) + .values({ + companyId, + agentId: recoveryAgent.id, + source: "automation", + triggerDetail: "system", + reason, + payload: withRecoveryContext({ issueId: issue.id, retryOfRunId: finishingRun.id }, "normal_model"), + status: "queued", + requestedByActorType: "system", + requestedByActorId: null, + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + await notifyRunDispatch(tx, { companyId }); + const queuedRun = await tx + .insert(heartbeatRuns) + .values({ + companyId, + agentId: recoveryAgent.id, + invocationSource: "automation", + triggerDetail: "system", + status: "queued", + wakeupRequestId: wakeupRequest.id, + contextSnapshot, + responsibleUserId, + sessionIdBefore: sessionBefore, + retryOfRunId: finishingRun.id, + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + await tx + .update(agentWakeupRequests) + .set({ runId: queuedRun.id, updatedAt: now }) + .where(and(eq(agentWakeupRequests.id, wakeupRequest.id), eq(agentWakeupRequests.companyId, companyId))); + + await tx + .update(issues) + .set({ + executionRunId: queuedRun.id, + executionAgentNameKey: normalizeAgentNameKey(recoveryAgent.name), + executionLockedAt: now, + updatedAt: now, + }) + .where(and(eq(issues.id, issue.id), eq(issues.companyId, companyId))); + + return toRunSummary(queuedRun); + }, + }; +} + +async function recordNativeTerminalRecoveryIfNeeded(tx: Db, run: ReleaseRun, issue: IssueRow, now: Date): Promise { + const applies = + run.runtimeMode === "native" && + ["failed", "timed_out", "interrupted", "cancelled"].includes(run.status) && + issue.assigneeAgentId === run.agentId && + !["done", "cancelled"].includes(issue.status); + if (!applies || isAcknowledgedNativeStop(run) || isAcknowledgedNativeReassignmentStop(run)) return false; + + const existing = await tx + .select({ id: issueRecoveryActions.id, evidence: issueRecoveryActions.evidence }) + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, issue.companyId), + eq(issueRecoveryActions.sourceIssueId, issue.id), + or( + inArray(issueRecoveryActions.status, ["active", "escalated"]), + sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'runId' = ${run.id}`, + ), + ), + ) + .limit(1); + let nativeFailureBlock: { runId: string; statusVersion: number } | undefined; + if (issue.status !== "blocked") { + const projected = await issueService(tx).update(issue.id, { status: "blocked" }, tx); + if (projected) { + nativeFailureBlock = { runId: run.id, statusVersion: projected.statusVersion }; + await notifyDeliveryWork(tx, DELIVERY_QUEUES.mcpEvents, `task:${issue.id}`); + await tx.insert(activityLog).values({ + companyId: issue.companyId, actorType: "system", actorId: "execution-recovery", + action: "issue.updated", entityType: "issue", entityId: issue.id, runId: run.id, + details: { status: "blocked", previousStatus: issue.status, reason: "native_continuation_requires_reconciliation" }, + }); + } + } + // Status projection is required even when restart/finalization created the + // incident first. Preserve its owner, cause, retry budget, and prior evidence. + if (nativeFailureBlock) { + for (const action of existing) { + await tx.update(issueRecoveryActions).set({ + evidence: { ...action.evidence, nativeFailureBlock }, updatedAt: now, + }).where(and(eq(issueRecoveryActions.id, action.id), eq(issueRecoveryActions.companyId, issue.companyId))); + } + } + if (!existing.length) { + await tx + .update(nativeRunFinalizations) + .set({ + phase: "terminal_failure", + leaseOwner: null, + leaseExpiresAt: null, + nextAttemptAt: null, + recoveryState: "blocked", + failureCode: "native_continuation_requires_reconciliation", + updatedAt: now, + }) + .where( + and( + eq(nativeRunFinalizations.companyId, issue.companyId), + eq(nativeRunFinalizations.runId, run.id), + isNull(nativeRunFinalizations.resultId), + ), + ); + await issueRecoveryActionService(tx).upsertSourceScoped({ + companyId: issue.companyId, + sourceIssueId: issue.id, + kind: "active_run_watchdog", + ownerType: "board", + returnOwnerAgentId: run.agentId, + cause: "native_continuation_requires_reconciliation", + fingerprint: `native-continuation:${run.id}`, + evidence: { runId: run.id, originalFailureCode: run.errorCode, ...(nativeFailureBlock ? { nativeFailureBlock } : {}) }, + nextAction: + "Inspect the original failure and reconcile the previous execution before continuing. Automatic recovery cannot start another incident.", + maxAttempts: 3, + wakePolicy: null, + supersedeOnIdentityChange: true, + }); + } + return true; +} + +/** + * Builds the temporary transaction-scope handle the admission port needs. + * `heartbeat.ts` calls this through `createWakeQueue`'s own wrapper; it + * never builds a `TransactionScope` itself. + */ +export function createAdmissionTransactionScope(companyId: string, tx: Db): TransactionScope { + return TransactionScope.create(companyId, tx); +} + +function requireAdmissionTx(scope: TransactionScope | null | undefined, companyId: string): Db { + return requireTransactionScopeTx(scope, companyId) as Db; +} + +export function createWakeAdmissionReader(): WakeAdmissionReader { + return { + async matchesActiveWakeActor(scope, input) { + const tx = requireAdmissionTx(scope, input.companyId); + if (!input.wakeupRequestId) return false; + const actor = await tx + .select({ + type: agentWakeupRequests.requestedByActorType, + id: agentWakeupRequests.requestedByActorId, + }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, input.companyId), + eq(agentWakeupRequests.id, input.wakeupRequestId), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + return ( + actor !== null && + actor.type === input.requestedByActorType && + actor.id === input.requestedByActorId + ); + }, + async isSameExecutionAgent( + scope, + { + companyId, + activeExecutionRunAgentId, + issueExecutionAgentNameKey, + agentNameKey, + }, + ) { + const tx = requireAdmissionTx(scope, companyId); + const executionAgent = await tx + .select({ name: agents.name }) + .from(agents) + .where( + and( + eq(agents.id, activeExecutionRunAgentId), + eq(agents.companyId, companyId), + ), + ) + .then((rows) => rows[0] ?? null); + const executionAgentNameKey = + normalizeAgentNameKey(issueExecutionAgentNameKey) ?? + normalizeAgentNameKey(executionAgent?.name); + return ( + Boolean(executionAgentNameKey) && executionAgentNameKey === agentNameKey + ); + }, + + async findExistingDeferredWake( + scope, + { companyId, agentId, issueId, chatConversationId, durableActor }, + ) { + const tx = requireAdmissionTx(scope, companyId); + const row = await tx + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.agentId, agentId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`, + ...(chatConversationId + ? [ + sql`coalesce(${agentWakeupRequests.payload} -> ${DEFERRED_WAKE_CONTEXT_KEY}::text ->> ${CHAT_CONVERSATION_ID_KEY}::text, ${chatConversationId}) = ${chatConversationId}`, + ] + : []), + ...(durableActor + ? [ + durableActor.type === null + ? isNull(agentWakeupRequests.requestedByActorType) + : eq( + agentWakeupRequests.requestedByActorType, + durableActor.type, + ), + durableActor.id === null + ? isNull(agentWakeupRequests.requestedByActorId) + : eq( + agentWakeupRequests.requestedByActorId, + durableActor.id, + ), + ] + : []), + ), + ) + .orderBy(asc(agentWakeupRequests.requestedAt)) + .limit(1) + .then((rows) => rows[0] ?? null); + if (!row) return null; + const payload = parseObject(row.payload); + return { + id: row.id, + runId: row.runId, + payload, + deferredContext: parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]), + coalescedCount: row.coalescedCount, + }; + }, + }; +} + +export function createWakeAdmissionWriter(): WakeAdmissionWriter { + return { + async coalesceIntoActiveExecutionRun(scope, input) { + const tx = requireAdmissionTx(scope, input.companyId); + const now = new Date(); + await notifyChatRunMilestones(tx, { companyId: input.companyId, runId: input.activeExecutionRunId }); + const mergedRun = await tx + .update(heartbeatRuns) + .set({ contextSnapshot: input.mergedContextSnapshot, updatedAt: now }) + .where( + and( + eq(heartbeatRuns.id, input.activeExecutionRunId), + eq(heartbeatRuns.companyId, input.companyId), + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + if (!mergedRun) { + // The compare-and-set write affected no row. Throw to roll the + // transaction back instead of recording a coalesced wake against a + // run this write never touched. + throw new Error( + "wake-queue: the coalesce target run was not found for this company", + ); + } + await tx.insert(agentWakeupRequests).values({ + ...input.durableReceipt, + companyId: input.companyId, + agentId: input.agentId, + source: input.source, + triggerDetail: input.triggerDetail, + reason: "issue_execution_same_name", + payload: input.payload, + status: "coalesced", + coalescedCount: 1, + requestedByActorType: input.requestedByActorType, + requestedByActorId: input.requestedByActorId, + idempotencyKey: input.idempotencyKey, + runId: mergedRun.id, + finishedAt: now, + }); + return mergedRun as unknown as Record; + }, + + async mergeIntoExistingDeferredWake(scope, input) { + const tx = requireAdmissionTx(scope, input.companyId); + if (input.manualUserWakeActorId) await notifyChatInboundWakeNotices(tx, { companyId: input.companyId }); + const [prior] = await tx.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.id, input.existingDeferredWakeId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + )).for("update"); + if (!prior) throw new Error("wake-queue: the deferred wake to merge into was not found for this company"); + const priorPayload = parseObject(prior.payload), issueId = readNonEmptyString(priorPayload.issueId); + const priorCommentId = readNonEmptyString(priorPayload.commentId), priorActor = readNonEmptyString(prior.requestedByActorId); + const [priorComment] = prior.requestedByActorType === "user" && priorActor && issueId && isUuidLike(issueId) && + priorCommentId && isUuidLike(priorCommentId) ? await tx.select({ id: issueComments.id, body: issueComments.body }).from(issueComments).where(and( + eq(issueComments.companyId, input.companyId), eq(issueComments.issueId, issueId), eq(issueComments.id, priorCommentId), + eq(issueComments.authorType, "user"), eq(issueComments.authorUserId, priorActor), + isNull(issueComments.deletedAt), isNull(issueComments.createdByRunId), + )) : []; + if ((input.coalescedReceipt || priorComment?.body.trim()) && + (readNonEmptyString(input.mergedPayload.issueId) !== issueId || (input.coalescedReceipt && + (input.coalescedReceipt.agentId !== prior.agentId || readNonEmptyString(input.coalescedReceipt.payload.issueId) !== issueId)))) + throw new Error("wake-queue: merged receipt does not match the saved queue scope"); + if (priorComment?.body.trim()) { + const [recorded] = await tx.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.agentId, prior.agentId), + eq(agentWakeupRequests.status, "coalesced"), eq(agentWakeupRequests.requestedByActorType, "user"), + eq(agentWakeupRequests.requestedByActorId, priorActor!), + prior.runId ? eq(agentWakeupRequests.runId, prior.runId) : isNull(agentWakeupRequests.runId), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, + sql`${agentWakeupRequests.payload}->>'commentId' = ${priorComment.id}`, + sql`${agentWakeupRequests.payload}->>'coalescedIntoWakeupRequestId' = ${prior.id}`, + )).limit(1); + if (!recorded) await tx.insert(agentWakeupRequests).values({ + companyId: input.companyId, agentId: prior.agentId, source: prior.source, triggerDetail: prior.triggerDetail, + reason: prior.reason, payload: { issueId, commentId: priorComment.id, coalescedIntoWakeupRequestId: prior.id }, + status: "coalesced", coalescedCount: 1, requestedByActorType: "user", requestedByActorId: priorActor, + requestedAt: prior.requestedAt, finishedAt: new Date(), runId: prior.runId, + }); + } + const rows = await tx + .update(agentWakeupRequests) + .set({ + payload: input.mergedPayload, + coalescedCount: input.nextCoalescedCount, + ...(input.manualUserWakeActorId ? { + requestedByActorType: "user", + requestedByActorId: input.manualUserWakeActorId, + } : {}), + updatedAt: new Date(), + }) + .where( + and( + eq(agentWakeupRequests.id, input.existingDeferredWakeId), + eq(agentWakeupRequests.companyId, input.companyId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + ), + ) + .returning({ id: agentWakeupRequests.id }); + if (rows.length === 0) { + // The compare-and-set write affected no row: a concurrent writer + // already moved this wake off `deferred_issue_execution`. Roll the + // transaction back instead of leaving the merge half-applied. + throw new Error( + "wake-queue: the deferred wake to merge into was not found for this company", + ); + } + if (input.coalescedReceipt) { + await tx.insert(agentWakeupRequests).values({ + ...input.coalescedReceipt, + companyId: input.companyId, + agentId: prior.agentId, + runId: prior.runId, + payload: { ...input.coalescedReceipt.payload, coalescedIntoWakeupRequestId: prior.id }, + status: "coalesced", + coalescedCount: 1, + finishedAt: new Date(), + }); + } + }, + + async insertNewDeferredWake(scope, input) { + const tx = requireAdmissionTx(scope, input.companyId); + await tx.insert(agentWakeupRequests).values({ + ...input.durableReceipt, + companyId: input.companyId, + agentId: input.agentId, + source: input.source, + triggerDetail: input.triggerDetail, + reason: "issue_execution_deferred", + payload: input.payload, + status: DEFERRED_WAKE_STATUS, + requestedByActorType: input.requestedByActorType, + requestedByActorId: input.requestedByActorId, + idempotencyKey: input.idempotencyKey, + }); + }, + }; +} + +export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAdapterDeps): IssueLockWriter { + return { + async withIssueExecutionLock(input, fn): Promise { + return db.transaction(async (rawTx) => { + const tx = rawTx as unknown as Db; + const evidence = await readReleaseRun(tx, input); + const run = evidence.run; + if (!run) { + throw new Error(`wake-queue: run ${input.runId} was not found while releasing issue execution`); + } + const runSnapshot = toRunSnapshot(run); + if (input.resumeStrandedQueue && (!run.finishedAt || !["succeeded", "failed", "cancelled", "timed_out", "interrupted"].includes(run.status))) { + return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } + const contextIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); + + // Lock the context issue (if any) and every issue that still references this + // run, in id order, so two concurrent finalizations can never deadlock on + // each other's row-lock acquisition order. + await tx.execute( + contextIssueId + ? sql` + select id from issues + where company_id = ${input.companyId} + and ( + id = ${contextIssueId} + or execution_run_id = ${run.id} + or checkout_run_id = ${run.id} + ) + order by id + for update + ` + : sql` + select id from issues + where company_id = ${input.companyId} + and (execution_run_id = ${run.id} or checkout_run_id = ${run.id}) + order by id + for update + `, + ); + + if (input.resumeStrandedQueue && run.runtimeMode === "native") { + const [finalized] = contextIssueId && run.nativeIssueId === contextIssueId + ? await tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.runId, run.id), + eq(nativeRunFinalizations.issueId, run.nativeIssueId), eq(nativeRunFinalizations.phase, "committed"), + )).limit(1) + : []; + if (!finalized) return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } + if (input.resumeStrandedQueue && await hasPendingRunCleanup(tx, input.companyId, run.id)) { + return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } + + const candidateIssues = await tx + .select() + .from(issues) + .where( + and( + eq(issues.companyId, input.companyId), + contextIssueId + ? or(eq(issues.id, contextIssueId), eq(issues.executionRunId, run.id), eq(issues.checkoutRunId, run.id)) + : or(eq(issues.executionRunId, run.id), eq(issues.checkoutRunId, run.id)), + ), + ) + .orderBy(asc(issues.id)); + + for (const issue of candidateIssues) { + if (issue.executionRunId === run.id || issue.checkoutRunId === run.id) { + await notifyExternalConversationSettlement(tx, { companyId: input.companyId, issueId: issue.id }); + await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: issue.id }); + await notifyNativeStatusReassessment(tx, { companyId: input.companyId, issueId: issue.id }); + } + } + // Two separate updates: a retry can move `executionRunId` to a new run + // while `checkoutRunId` still points at this one finishing. + await tx + .update(issues) + .set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, updatedAt: input.now }) + .where(and(eq(issues.companyId, input.companyId), eq(issues.executionRunId, run.id))); + await tx + .update(issues) + .set({ checkoutRunId: null, updatedAt: input.now }) + .where(and(eq(issues.companyId, input.companyId), eq(issues.checkoutRunId, run.id))); + + const issueRow = + (contextIssueId ? candidateIssues.find((candidate) => candidate.id === contextIssueId) : candidateIssues[0]) ?? null; + + // A queue interrupt authorizes only its original pending queue. Replays + // after dispatch or deleting the final message cannot launch other work. + const interruptQueueId = run.runtimeMode !== "native" + ? readNonEmptyString(run.resultJson?.queuedCommentInterruptQueueId) + : null; + const [interruptedQueue] = interruptQueueId && issueRow + ? await tx.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.id, interruptQueueId), + eq(agentWakeupRequests.companyId, run.companyId), + eq(agentWakeupRequests.agentId, run.agentId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issueRow.id}`, + )).limit(1) + : []; + const cancellation = readRunCancellation(run.resultJson); + const queuedOperatorStop = run.runtimeMode === "legacy" && run.status === "cancelled" && + run.runtimeModeResolvedAt === null && run.startedAt === null && run.processStartedAt === null && + run.processPid === null && run.processGroupId === null && + cancellation?.source === "operator" && cancellation.expected; + const preDrainFacts: PreDrainFacts = { + issueRowPresent: issueRow !== null, + executionRunIdMatchesRun: !issueRow || !issueRow.executionRunId || issueRow.executionRunId === run.id, + isWorkspaceValidationFailedRun: isWorkspaceValidationFailedRun(run), + isConfigurationIncompleteFailedRun: isConfigurationIncompleteFailedRun(run), + issueStatus: issueRow?.status ?? "", + hasAssigneeUser: Boolean(issueRow?.assigneeUserId), + assigneeAgentMatchesRunAgent: issueRow?.assigneeAgentId === run.agentId, + legacyExecutionNeedsReconciliation: evidence.kind === "complete" && + await legacyExecutionNeedsReconciliationWithEvidence(tx as unknown as Db, evidence.run), + // An operator stop never promotes old queued work by itself. The + // next explicit wake adopts those messages atomically when it + // queues a run. + executionCancellationAcknowledged: + isAcknowledgedNativeReassignmentStop(run) || + (run.status === "cancelled" && + (queuedOperatorStop || parseObject(run.resultJson?.executionCancellation).state === "acknowledged" || isAcknowledgedNativeStop(run)) && + !interruptedQueue), + }; + const preDrain = decidePreDrain(preDrainFacts); + + if (preDrain.kind === "released") { + return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } + + // decidePreDrain only returns "blocked" or "proceed" when the issue row is present. + if (!issueRow) { + throw new Error(`wake-queue: pre-drain decision ${preDrain.kind} reached without an issue row`); + } + + // Enqueue does not stamp executionRunId until dispatch. A concurrent + // queued successor still owns the next turn, including during a late + // finalization/stranded-queue retry under this issue lock. Another + // agent's review participation retains its separate recovery path. + const [successor] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, input.companyId), + input.resumeStrandedQueue ? undefined : eq(heartbeatRuns.agentId, run.agentId), + input.resumeStrandedQueue ? undefined : sql`${heartbeatRuns.id} <> ${run.id}`, + or(eq(heartbeatRuns.nativeIssueId, issueRow.id), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueRow.id}`), + or(inArray(heartbeatRuns.status, ["queued", "running", "scheduled_retry"]), + input.resumeStrandedQueue ? sql`(${heartbeatRuns.createdAt}, ${heartbeatRuns.id}) > ( + SELECT source.created_at, source.id FROM heartbeat_runs source + WHERE source.company_id = ${input.companyId} AND source.id = ${run.id})` : undefined), + )).limit(1); + if (successor) return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + + if (preDrain.kind === "blocked") { + return { + outcome: { + kind: "blocked", + issue: toIssueSnapshot(issueRow), + previousStatus: issueRow.status as "todo" | "in_progress", + noticeKind: preDrain.noticeKind, + }, + postCommitEffects: [], + run: runSnapshot, + }; + } + + // The durable external answer owns the sole successor of this retired + // question source; release locks without creating another incident. + if ( + run.runtimeMode === "native" && + run.status === "cancelled" && + run.errorCode === "external_chat_continuation" && + (await isRetiredExternalChatQuestionSource(tx, { + companyId: run.companyId, + issueId: issueRow.id, + agentId: run.agentId, + runId: run.id, + })) + ) { + return { + outcome: { kind: "released" }, + postCommitEffects: [], + run: runSnapshot, + }; + } + + if (await recordNativeTerminalRecoveryIfNeeded(tx, run, issueRow, input.now)) { + return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } + + // A release must leave deferred messages intact while execution is held. + // The finishing conversation may still own its lease until finally cleanup. + // Allow only bounded retry planning in that case; admission stays gated. + const executionBlocker = await getExecutionBlocker(tx, issueRow.companyId, issueRow.id); + const recoveryOnly = Boolean(executionBlocker && + executionBlocker.cause === "execution_owner_active" && executionBlocker.runId === run.id && + runSnapshot.conversationContinuation && ["failed", "timed_out", "interrupted"].includes(run.status)); + if (executionBlocker && !recoveryOnly) { + return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } + + // Releases still settle while Agent Chat is disabled, but no deferred + // turn or recovery successor may be created. Check here in the shared + // transaction so cleanup retries and restart sweeps use the same gate. + if (issueRow.conversationAgentId && !(await instanceSettingsService(tx).getExperimental()).enableAgentChat) { + return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } + + const locked: LockedIssueExecution = { primaryIssue: toIssueSnapshot(issueRow), run: runSnapshot, recoveryOnly }; + const result = await fn(locked, { host: buildHost(tx, deps), transaction: buildTransaction(tx, deps, db, run) }); + // Explicit retry admission requires the exact source's task claim. + // Preserve it only when the existing queue-first recovery policy + // requests that retry. The clear/restore stays inside this task lock. + if (run.runtimeMode === "legacy" && ["failed", "timed_out"].includes(run.status) && + run.contextSnapshot?.explicitUserContinuation && issueRow.executionRunId === run.id && + result.postCommitEffects.some(effect => effect.kind === "conversation_retry_requested" && effect.runId === run.id)) { + await tx.update(issues).set({ executionRunId: run.id, + executionAgentNameKey: issueRow.executionAgentNameKey, + executionLockedAt: issueRow.executionLockedAt, + }).where(and(eq(issues.companyId, input.companyId), eq(issues.id, issueRow.id), + sql`${issues.executionRunId} is null`)); + } + return { ...result, run: runSnapshot }; + }); + }, + }; +} diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 0eff1c83613..cc4cf161464 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -1 +1,1158 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyApprovalWork } from "./approval-work-notifications.js"; +import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; +import { addCents, compareCents, normalizeCents } from "@paperclipai/shared"; +import { and, desc, eq, inArray, sql, or, isNull } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; + +import { + agents, + approvals, + budgetIncidents, + budgetPolicies, + budgetReservations, + companies, + costEvents, + heartbeatRuns, + projects, +} from "@paperclipai/db"; +import type { + BudgetIncident, + BudgetIncidentResolutionInput, + BudgetMetric, + BudgetOverview, + PauseReason, + BudgetPolicy, + BudgetPolicySummary, + BudgetPolicyUpsertInput, + BudgetScopeType, + BudgetThresholdType, + BudgetWindowKind, +} from "@paperclipai/shared"; +import { HttpError, notFound, unprocessable } from "../errors.js"; +import { logger } from "../middleware/logger.js"; +import { withAccountingTransaction } from "./accounting-transaction.js"; +import { logActivity, type LogActivityInput, type ActivityPublication } from "./activity-log.js"; +import { computeObservedSpend, computeObservedSpends, type ObservedSpend } from "./budget-exposure.js"; +import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { listPendingBudgetCancellationScopes } from "./heartbeat/budget-cancellation.js"; +import { withBudgetSpan } from "./budget-tracing.js"; +export { computeObservedSpend, computeObservedSpends } from "./budget-exposure.js"; +export type { BudgetExposure } from "./budget-exposure.js"; + +type ScopeRecord = { + companyId: string; + name: string; + paused: boolean; + pauseReason: PauseReason | null; + budgetHeld: boolean; +}; + +type PolicyRow = typeof budgetPolicies.$inferSelect; +type IncidentRow = typeof budgetIncidents.$inferSelect; + +export type BudgetEnforcementScope = { + companyId: string; + scopeType: BudgetScopeType; + scopeId: string; + /** Snapshot taken under the company lock, before a later budget grant can admit new work. */ + createdBefore?: Date; + /** Revalidated at the durable cancellation write, under the admission lock. */ + enforcement?: { policyId: string; version: number }; +}; + +export type BudgetServiceHooks = { + cancelWorkForScope?: (scope: BudgetEnforcementScope) => Promise; +}; + +function currentUtcMonthWindow(now = new Date()) { + const year = now.getUTCFullYear(); + const month = now.getUTCMonth(); + const start = new Date(Date.UTC(year, month, 1, 0, 0, 0, 0)); + const end = new Date(Date.UTC(year, month + 1, 1, 0, 0, 0, 0)); + return { start, end }; +} + +function resolveWindow(windowKind: BudgetWindowKind, now = new Date()) { + if (windowKind === "lifetime") { + return { + start: new Date(Date.UTC(1970, 0, 1, 0, 0, 0, 0)), + end: new Date(Date.UTC(9999, 0, 1, 0, 0, 0, 0)), + }; + } + return currentUtcMonthWindow(now); +} + +function budgetStatusFromObserved( + observedAmount: string, + amount: number, + warnPercent: number, +): BudgetPolicySummary["status"] { + if (amount <= 0) return "ok"; + if (compareCents(observedAmount, amount) >= 0) return "hard_stop"; + if (compareCents(observedAmount, Math.ceil((amount * warnPercent) / 100)) >= 0) return "warning"; + return "ok"; +} + +function normalizeScopeName(scopeType: BudgetScopeType, name: string) { + if (scopeType === "company") return name; + return name.trim().length > 0 ? name : scopeType; +} + +async function resolveScopeRecord(db: Db, scopeType: BudgetScopeType, scopeId: string): Promise { + if (scopeType === "company") { + const row = await db + .select({ + companyId: companies.id, + name: companies.name, + status: companies.status, + pauseReason: companies.pauseReason, + pausedAt: companies.pausedAt, + }) + .from(companies) + .where(eq(companies.id, scopeId)) + .then((rows) => rows[0] ?? null); + if (!row) throw notFound("Company not found"); + return { + companyId: row.companyId, + name: row.name, + paused: row.status === "paused" || Boolean(row.pausedAt), + pauseReason: (row.pauseReason as ScopeRecord["pauseReason"]) ?? null, + budgetHeld: row.pauseReason === "budget", + }; + } + + if (scopeType === "agent") { + const row = await db + .select({ + companyId: agents.companyId, + name: agents.name, + status: agents.status, + pauseReason: agents.pauseReason, + lifecycleHolds: agents.lifecycleHolds, + }) + .from(agents) + .where(eq(agents.id, scopeId)) + .then((rows) => rows[0] ?? null); + if (!row) throw notFound("Agent not found"); + return { + companyId: row.companyId, + name: row.name, + paused: row.status === "paused", + pauseReason: (row.pauseReason as ScopeRecord["pauseReason"]) ?? null, + budgetHeld: row.lifecycleHolds.includes("budget"), + }; + } + + const row = await db + .select({ + companyId: projects.companyId, + name: projects.name, + pauseReason: projects.pauseReason, + pausedAt: projects.pausedAt, + }) + .from(projects) + .where(eq(projects.id, scopeId)) + .then((rows) => rows[0] ?? null); + if (!row) throw notFound("Project not found"); + return { + companyId: row.companyId, + name: row.name, + paused: Boolean(row.pausedAt), + pauseReason: (row.pauseReason as ScopeRecord["pauseReason"]) ?? null, + budgetHeld: row.pauseReason === "budget", + }; +} + +// Keep a large dashboard from filling the database pool with queued reads. +async function mapBudgetReads(rows: T[], read: (row: T) => Promise): Promise { + const result: R[] = []; + for (let start = 0; start < rows.length; start += 4) { + result.push(...await Promise.all(rows.slice(start, start + 4).map(read))); + } + return result; +} + +/** Filter in PostgreSQL so one run never loads unrelated agents' policies. */ +export function budgetPoliciesForRun(companyId: string, agentId: string | null, projectId: string | null) { + return or( + and(eq(budgetPolicies.scopeType, "company"), eq(budgetPolicies.scopeId, companyId)), + agentId ? and(eq(budgetPolicies.scopeType, "agent"), eq(budgetPolicies.scopeId, agentId)) : undefined, + projectId ? and(eq(budgetPolicies.scopeType, "project"), eq(budgetPolicies.scopeId, projectId)) : undefined, + ); +} + +function unpricedUsageBlocks(policy: PolicyRow, observed: ObservedSpend) { + return policy.unpricedUsagePolicy !== "allow" + && (observed.unpricedEventCount > 0 || observed.unknownBrowserRunCount > 0); +} + +function observedBlocks(policy: PolicyRow, observed: Awaited>) { + if (!policy.isActive || !policy.hardStopEnabled || policy.amount <= 0) return false; + return compareCents(observed.totalExact, policy.amount) >= 0 || (observed.pendingRunCount - observed.recoveringRunCount) > 0 || unpricedUsageBlocks(policy, observed); +} +export async function policyBlocks(db: Db, policy: PolicyRow) { + if (!policy.isActive || !policy.hardStopEnabled || policy.amount <= 0) return false; + return observedBlocks(policy, await computeObservedSpend(db, policy)); +} + +/** Supplemental handoff check after authoritative accounting admission commits. + * Read only: the caller can hold issue/run ownership, so this must never take + * the company lock, reconcile holds, or deliver effects. A later concurrent + * charge targets already-admitted work through the normal durable Stop path. */ +export async function recheckAdmittedRunBudgetReadOnly(db: Db, input: { + companyId: string; agentId: string; runId: string; projectId: string | null; reservationId: string; +}): Promise { + return withBudgetSpan("budget.admission.evaluate", async () => { + const [reservation] = await db.select({ amountCents: budgetReservations.amountCents }).from(budgetReservations).where(and( + eq(budgetReservations.id, input.reservationId), eq(budgetReservations.companyId, input.companyId), + eq(budgetReservations.runId, input.runId), eq(budgetReservations.agentId, input.agentId), + sql`${budgetReservations.projectId} is not distinct from ${input.projectId}::uuid`, + eq(budgetReservations.state, "held"), + )); + if (!reservation) return "Provider dispatch no longer owns a held budget reservation"; + const policies = await db.select().from(budgetPolicies).where(and(eq(budgetPolicies.companyId, input.companyId), + eq(budgetPolicies.isActive, true), eq(budgetPolicies.hardStopEnabled, true), eq(budgetPolicies.metric, "billed_cents"), + sql`${budgetPolicies.amount} > 0`, budgetPoliciesForRun(input.companyId, input.agentId, input.projectId))); + const observations = await computeObservedSpends(db, policies); + for (const [index, policy] of policies.entries()) { + const observed = observations[index]; + if (observedBlocks(policy, observed)) return "Applicable budget no longer permits provider work"; + const capacity = compareCents(addCents(observed.ledgerCents, observed.effectiveHeldCents), policy.amount); + if (capacity > 0 || (compareCents(reservation.amountCents, 0) === 0 && capacity >= 0)) + return "Available budget is reserved by unfinished runs"; + } + return null; + }); +} + +function buildApprovalPayload(input: { + policy: PolicyRow; + scopeName: string; + thresholdType: BudgetThresholdType; + amountObserved: number; + amountObservedExact: string; + windowStart: Date; + windowEnd: Date; +}) { + return { + scopeType: input.policy.scopeType, + scopeId: input.policy.scopeId, + scopeName: input.scopeName, + metric: input.policy.metric, + windowKind: input.policy.windowKind, + thresholdType: input.thresholdType, + budgetAmount: input.policy.amount, + observedAmount: input.amountObserved, + observedAmountExact: input.amountObservedExact, + warnPercent: input.policy.warnPercent, + windowStart: input.windowStart.toISOString(), + windowEnd: input.windowEnd.toISOString(), + policyId: input.policy.id, + guidance: "Resolve pending or unpriced accounting, raise the budget, or keep the scope paused.", + }; +} + +async function markApprovalStatus( + db: Db, + approvalId: string | null, + status: "approved" | "rejected", + decisionNote: string | null | undefined, + decidedByUserId: string, +) { + if (!approvalId) return; + const [approval] = await db.select({ companyId: approvals.companyId }).from(approvals).where(eq(approvals.id, approvalId)); + if (!approval) return; + await notifyApprovalWork(db, approval.companyId, { approvalId }); + await db + .update(approvals) + .set({ + status, + decisionNote: decisionNote ?? null, + decidedByUserId, + decidedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(approvals.id, approvalId)); +} + +/** Internal: caller must hold the company accounting transaction lock. */ +export function budgetServiceInTransaction(db: Db, publications: ActivityPublication[] = []) { + const recordActivity = (input: LogActivityInput) => logActivity(db, input, publications); + async function pauseScopeForBudget(policy: PolicyRow) { + const now = new Date(); + if (policy.scopeType === "agent") { + const { applyAgentBudgetHoldInTransaction } = await import("./agent-lifecycle.js"); + await applyAgentBudgetHoldInTransaction(db, policy.companyId, policy.scopeId, true); + return; + } + + if (policy.scopeType === "project") { + await notifyRunDispatch(db, { companyId: policy.companyId }); + await db + .update(projects) + .set({ + pauseReason: "budget", + pausedAt: now, + updatedAt: now, + }) + .where(and(eq(projects.id, policy.scopeId), isNull(projects.pausedAt))); + return; + } + + const scope = await resolveScopeRecord(db, "company", policy.scopeId); + if (scope.paused) return; + await notifyRunDispatch(db, { companyId: policy.companyId }); + await notifyCompanyRetentionWork(db, policy.companyId); + await notifyStatusCardIssueChange(db, policy.companyId); + await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); + await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); + await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors); + await notifyDeliveryWork(db, DELIVERY_QUEUES.agentLifecycle, policy.companyId); + await notifyDeliveryWork(db, DELIVERY_QUEUES.skillSourceSync); + await db + .update(companies) + .set({ + status: "paused", + pauseReason: "budget", + pausedAt: now, + updatedAt: now, + }) + .where(and(eq(companies.id, policy.scopeId), eq(companies.status, "active"))); + } + + async function pauseAndCancelScopeForBudget(policy: PolicyRow, requestCancellation = true) { + await pauseScopeForBudget(policy); + if (!requestCancellation) return; + // A pending generation already covers all admitted work in this scope. + // Rechecks must not invalidate a worker that is delivering that intent. + await notifyDeliveryWork(db, DELIVERY_QUEUES.budgetEnforcement, policy.companyId); + await db.update(budgetPolicies).set({ + enforcementVersion: sql`case when ${budgetPolicies.enforcementVersion} > ${budgetPolicies.enforcementDeliveredVersion} + then ${budgetPolicies.enforcementVersion} else ${budgetPolicies.enforcementVersion} + 1 end`, + updatedAt: sql`case when ${budgetPolicies.enforcementVersion} > ${budgetPolicies.enforcementDeliveredVersion} + then ${budgetPolicies.updatedAt} else now() end`, + }).where(eq(budgetPolicies.id, policy.id)); + } + + async function resumeScopeFromBudget(policy: PolicyRow) { + const policies = await db.select().from(budgetPolicies).where(and( + eq(budgetPolicies.companyId, policy.companyId), eq(budgetPolicies.scopeType, policy.scopeType), + eq(budgetPolicies.scopeId, policy.scopeId), eq(budgetPolicies.isActive, true), eq(budgetPolicies.hardStopEnabled, true), + )); + const observations = await computeObservedSpends(db, policies); + if (policies.some((candidate, index) => observedBlocks(candidate, observations[index]))) return; + const now = new Date(); + if (policy.scopeType === "agent") { + const { applyAgentBudgetHoldInTransaction } = await import("./agent-lifecycle.js"); + await applyAgentBudgetHoldInTransaction(db, policy.companyId, policy.scopeId, false); + return; + } + + if (policy.scopeType === "project") { + await notifyRunDispatch(db, { companyId: policy.companyId }); + await db + .update(projects) + .set({ + pauseReason: null, + pausedAt: null, + updatedAt: now, + }) + .where(and(eq(projects.id, policy.scopeId), eq(projects.pauseReason, "budget"))); + return; + } + + await notifyDeliveryWork(db, DELIVERY_QUEUES.skillSourceSync); + const scope = await resolveScopeRecord(db, "company", policy.scopeId); + if (scope.pauseReason !== "budget") return; + await notifyRunDispatch(db, { companyId: policy.companyId }); + await notifyCompanyRetentionWork(db, policy.companyId); + await notifyStatusCardIssueChange(db, policy.companyId); + await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); + await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); + await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors); + await notifyDeliveryWork(db, DELIVERY_QUEUES.agentLifecycle, policy.companyId); + await db + .update(companies) + .set({ + status: "active", + pauseReason: null, + pausedAt: null, + updatedAt: now, + }) + .where(and(eq(companies.id, policy.scopeId), eq(companies.status, "paused"), eq(companies.pauseReason, "budget"))); + } + + async function getPolicyRow(policyId: string) { + const policy = await db + .select() + .from(budgetPolicies) + .where(eq(budgetPolicies.id, policyId)) + .then((rows) => rows[0] ?? null); + if (!policy) throw notFound("Budget policy not found"); + return policy; + } + + async function listPolicyRows(companyId: string) { + return db + .select() + .from(budgetPolicies) + .where(eq(budgetPolicies.companyId, companyId)) + .orderBy(desc(budgetPolicies.updatedAt)); + } + + async function buildPolicySummary(policy: PolicyRow): Promise { + const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId); + const observed = await computeObservedSpend(db, policy); + const { total: observedAmount, totalExact: observedAmountExact, unpricedEventCount, pendingRunCount } = observed; + const { windowStart: start, windowEnd: end } = observed; + const amount = policy.isActive ? policy.amount : 0; + const utilizationPercent = + amount > 0 ? Number(((observedAmount / amount) * 100).toFixed(2)) : 0; + return { + policyId: policy.id, + companyId: policy.companyId, + scopeType: policy.scopeType as BudgetScopeType, + scopeId: policy.scopeId, + scopeName: normalizeScopeName(policy.scopeType as BudgetScopeType, scope.name), + metric: policy.metric as BudgetMetric, + windowKind: policy.windowKind as BudgetWindowKind, + amount, + observedAmount, + observedAmountExact, + reservationCents: policy.reservationCents, + unpricedEventCount, + pendingRunCount, + unpricedUsagePolicy: policy.unpricedUsagePolicy as "block" | "allow", + remainingAmount: amount > 0 ? Math.max(0, amount - observedAmount) : 0, + utilizationPercent, + warnPercent: policy.warnPercent, + hardStopEnabled: policy.hardStopEnabled, + notifyEnabled: policy.notifyEnabled, + isActive: policy.isActive, + status: observedBlocks(policy, observed) ? "hard_stop" : policy.isActive + ? budgetStatusFromObserved(observedAmountExact, amount, policy.warnPercent) + : "ok", + paused: scope.paused, + pauseReason: scope.pauseReason, + windowStart: start, + windowEnd: end, + }; + } + + async function createIncidentIfNeeded( + policy: PolicyRow, + thresholdType: BudgetThresholdType, + observed: Awaited>, + ) { + const { windowStart: start, windowEnd: end } = observed; + const existing = await db + .select() + .from(budgetIncidents) + .where( + and( + eq(budgetIncidents.policyId, policy.id), + eq(budgetIncidents.windowStart, start), + eq(budgetIncidents.thresholdType, thresholdType), + or(eq(budgetIncidents.status, "open"), and(eq(budgetIncidents.status, "dismissed"), eq(budgetIncidents.amountLimit, policy.amount))), + ), + ) + .orderBy(desc(budgetIncidents.createdAt)) + .then((rows) => rows[0] ?? null); + if (existing) return { incident: existing, created: false }; + + const { total: amountObserved, totalExact: amountObservedExact } = observed; + const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId); + const payload = buildApprovalPayload({ + policy, + scopeName: normalizeScopeName(policy.scopeType as BudgetScopeType, scope.name), + thresholdType, + amountObserved, + amountObservedExact, + windowStart: start, + windowEnd: end, + }); + + if (thresholdType === "hard") await notifyApprovalWork(db, policy.companyId); + const approval = thresholdType === "hard" + ? await db + .insert(approvals) + .values({ + companyId: policy.companyId, + type: "budget_override_required", + requestedByUserId: null, + requestedByAgentId: null, + status: "pending", + payload, + }) + .returning() + .then((rows) => rows[0] ?? null) + : null; + + const incident = await db + .insert(budgetIncidents) + .values({ + companyId: policy.companyId, + policyId: policy.id, + scopeType: policy.scopeType, + scopeId: policy.scopeId, + metric: policy.metric, + windowKind: policy.windowKind, + windowStart: start, + windowEnd: end, + thresholdType, + amountLimit: policy.amount, + amountObserved: sql`${amountObservedExact}::numeric`, + status: "open", + approvalId: approval?.id ?? null, + }) + .returning() + .then((rows) => rows[0] ?? null); + return incident ? { incident, created: true } : null; + } + + async function resolveOpenSoftIncidents(policyId: string) { + await db + .update(budgetIncidents) + .set({ + status: "resolved", + resolvedAt: new Date(), + updatedAt: new Date(), + }) + .where( + and( + eq(budgetIncidents.policyId, policyId), + eq(budgetIncidents.thresholdType, "soft"), + eq(budgetIncidents.status, "open"), + ), + ); + } + + async function resolveOpenIncidentsForPolicy( + policyId: string, + approvalStatus: "approved" | "rejected" | null, + decidedByUserId: string | null, + thresholdType?: BudgetThresholdType, + ) { + const condition = and(eq(budgetIncidents.policyId, policyId), eq(budgetIncidents.status, "open"), + thresholdType ? eq(budgetIncidents.thresholdType, thresholdType) : undefined); + const openRows = await db + .select() + .from(budgetIncidents) + .where(condition); + + await db + .update(budgetIncidents) + .set({ + status: "resolved", + resolvedAt: new Date(), + updatedAt: new Date(), + }) + .where(condition); + + if (!approvalStatus || !decidedByUserId) return; + for (const row of openRows) { + await markApprovalStatus(db, row.approvalId ?? null, approvalStatus, "Resolved via budget update", decidedByUserId); + } + } + + async function hydrateIncidentRows(rows: IncidentRow[]): Promise { + const approvalIds = rows.map((row) => row.approvalId).filter((value): value is string => Boolean(value)); + const approvalRows = approvalIds.length > 0 + ? await db + .select({ id: approvals.id, status: approvals.status }) + .from(approvals) + .where(inArray(approvals.id, approvalIds)) + : []; + const approvalStatusById = new Map(approvalRows.map((row) => [row.id, row.status])); + + return mapBudgetReads(rows, async (row) => { + const scope = await resolveScopeRecord(db, row.scopeType as BudgetScopeType, row.scopeId); + return { + id: row.id, + companyId: row.companyId, + policyId: row.policyId, + scopeType: row.scopeType as BudgetScopeType, + scopeId: row.scopeId, + scopeName: normalizeScopeName(row.scopeType as BudgetScopeType, scope.name), + metric: row.metric as BudgetMetric, + windowKind: row.windowKind as BudgetWindowKind, + windowStart: row.windowStart, + windowEnd: row.windowEnd, + thresholdType: row.thresholdType as BudgetThresholdType, + amountLimit: row.amountLimit, + amountObserved: row.amountObserved, + status: row.status as BudgetIncident["status"], + approvalId: row.approvalId ?? null, + approvalStatus: row.approvalId ? approvalStatusById.get(row.approvalId) ?? null : null, + resolvedAt: row.resolvedAt ?? null, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; + }); + } + + async function reconcileScope(companyId: string, scopeType: BudgetScopeType, scopeId: string, + prepared?: { scope: ScopeRecord; policies: PolicyRow[]; observations: ObservedSpend[] }) { + const scope = prepared?.scope ?? await resolveScopeRecord(db, scopeType, scopeId); + if (scope.companyId !== companyId) throw notFound("Budget scope not found"); + const policies = prepared?.policies ?? await db.select().from(budgetPolicies).where(and( + eq(budgetPolicies.companyId, companyId), eq(budgetPolicies.scopeType, scopeType), eq(budgetPolicies.scopeId, scopeId), + )); + let recoveringPolicy: PolicyRow | undefined; + const observations = prepared?.observations ?? await computeObservedSpends(db, policies); + for (const [index, policy] of policies.entries()) { + const observed = observations[index]; + if (observedBlocks(policy, observed)) { + await createIncidentIfNeeded(policy, "hard", observed); + await pauseAndCancelScopeForBudget(policy, !scope.budgetHeld); + const title = scopeType === "company" ? "Company" : scopeType === "agent" ? "Agent" : "Project"; + if (observed.pendingRunCount > 0) { + return { scopeType, scopeId, scopeName: scope.name, reason: `${title} cannot start work while completed runs await accounting.` }; + } + if (unpricedUsageBlocks(policy, observed)) { + return { scopeType, scopeId, scopeName: scope.name, reason: `${title} cannot start work because recorded usage has no reliable price.` }; + } + return { scopeType, scopeId, scopeName: scope.name, reason: scopeType === "project" + ? "Project cannot start work because its budget hard-stop is still exceeded." + : `${title} is paused because its budget hard-stop was reached.` }; + } + // An open native coordinator must finish the original run's accounting. + // Deny fresh work without pausing/cancelling that recovery or its hold. + if (policy.isActive && policy.hardStopEnabled && policy.amount > 0 && observed.recoveringRunCount > 0) recoveringPolicy ??= policy; + // A closed UTC window, inactive policy, or budget raise releases only + // budget-owned pauses. Manual pauses remain operator decisions. + await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "hard"); + if (!policy.isActive || !policy.notifyEnabled || compareCents(observed.totalExact, Math.ceil(policy.amount * policy.warnPercent / 100)) < 0) { + await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "soft"); + } + } + // Agent status has its own admission gate and error code. Preserve a + // manual pause without misclassifying it as a budget failure. + if (scopeType !== "agent" && scope.paused && scope.pauseReason !== "budget") { + return { scopeType, scopeId, scopeName: scope.name, reason: `${scopeType === "company" ? "Company" : "Project"} is paused and cannot start new work.` }; + } + if (scope.paused && scope.pauseReason === "budget" && !policies.length) { + return { scopeType, scopeId, scopeName: scope.name, reason: "Budget pause requires a policy or an explicit operator resume." }; + } + if (scope.budgetHeld && policies[0]) await resumeScopeFromBudget(policies[0]); + if (recoveringPolicy) return { scopeType, scopeId, scopeName: scope.name, + reason: "New work must wait while a native run recovers its unfinished accounting." }; + return null; + } + + async function evaluateAffectedScopes(companyId: string, agentId: string | null, projectId: string | null) { + const candidatePolicies = await db + .select() + .from(budgetPolicies) + .where( + and( + eq(budgetPolicies.companyId, companyId), + eq(budgetPolicies.isActive, true), + budgetPoliciesForRun(companyId, agentId, projectId), + ), + ); + + const policies = candidatePolicies.filter(policy => policy.metric === "billed_cents" && policy.amount > 0); + const observations = await computeObservedSpends(db, policies); + for (const [index, policy] of policies.entries()) { + const observed = observations[index]; + const observedAmount = observed.total; + const softThreshold = Math.ceil((policy.amount * policy.warnPercent) / 100); + + if (policy.notifyEnabled && compareCents(observed.totalExact, softThreshold) >= 0 && (!policy.hardStopEnabled || compareCents(observed.totalExact, policy.amount) < 0)) { + const softIncident = await createIncidentIfNeeded(policy, "soft", observed); + if (softIncident?.created) { + await recordActivity({ + companyId: policy.companyId, + actorType: "system", + actorId: "budget_service", + action: "budget.soft_threshold_crossed", + entityType: "budget_incident", + entityId: softIncident.incident.id, + details: { + scopeType: policy.scopeType, + scopeId: policy.scopeId, + amountObserved: observedAmount, + amountLimit: policy.amount, + }, + }); + } + } + + if (observedBlocks(policy, observed)) { + await resolveOpenSoftIncidents(policy.id); + const hardIncident = await createIncidentIfNeeded(policy, "hard", observed); + await pauseAndCancelScopeForBudget(policy); + if (hardIncident?.created) { + await recordActivity({ + companyId: policy.companyId, + actorType: "system", + actorId: "budget_service", + action: "budget.hard_threshold_crossed", + entityType: "budget_incident", + entityId: hardIncident.incident.id, + details: { + scopeType: policy.scopeType, + scopeId: policy.scopeId, + amountObserved: observedAmount, + amountLimit: policy.amount, + approvalId: hardIncident.incident.approvalId ?? null, + }, + }); + } + } + } + // A final receipt or price correction can reduce a partial observation. + // Release only this scope's budget-owned hold after all its policies agree. + const healthyScopes = new Map(); + for (const [index, policy] of policies.entries()) { + if (!observedBlocks(policy, observations[index])) healthyScopes.set(`${policy.scopeType}:${policy.scopeId}`, policy); + } + for (const policy of healthyScopes.values()) { + const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId); + if (scope.budgetHeld) await resumeScopeFromBudget(policy); + } + } + + async function evaluateAdmissionInTransaction(companyId: string, agentId: string | null, + context?: { issueId?: string | null; projectId?: string | null }) { + const scopes: Array<{ scopeType: BudgetScopeType; scopeId: string }> = [ + { scopeType: "company", scopeId: companyId }, ...(agentId ? [{ scopeType: "agent" as const, scopeId: agentId }] : []), + ...(context?.projectId ? [{ scopeType: "project" as const, scopeId: context.projectId }] : []), + ]; + const records = await Promise.all(scopes.map(scope => resolveScopeRecord(db, scope.scopeType, scope.scopeId))); + if (records.some(scope => scope.companyId !== companyId)) throw notFound("Budget scope not found"); + const policies = await db.select().from(budgetPolicies).where(and(eq(budgetPolicies.companyId, companyId), + budgetPoliciesForRun(companyId, agentId, context?.projectId ?? null))); + const observations = await computeObservedSpends(db, policies); + const reservationIndexes = policies.map((policy, i) => policy.isActive && policy.hardStopEnabled && policy.metric === "billed_cents" && policy.amount > 0 ? i : -1).filter(i => i >= 0); + const admission = (block: Awaited>) => ({ block, + policies: reservationIndexes.map(i => policies[i]), observations: reservationIndexes.map(i => observations[i]) }); + for (const [index, { scopeType, scopeId }] of scopes.entries()) { + const indexes = policies.map((policy, i) => policy.scopeType === scopeType && policy.scopeId === scopeId ? i : -1).filter(i => i >= 0); + const block = await reconcileScope(companyId, scopeType, scopeId, { scope: records[index], + policies: indexes.map(i => policies[i]), observations: indexes.map(i => observations[i]) }); + if (block) return admission(block); + } + return admission(null); + } + + const evaluateAdmission = (...args: Parameters) => + withBudgetSpan("budget.admission.evaluate", () => evaluateAdmissionInTransaction(...args)); + + return { + reconcileScope, + listPolicies: async (companyId: string): Promise => { + const rows = await listPolicyRows(companyId); + return rows.map((row) => ({ + ...row, + scopeType: row.scopeType as BudgetScopeType, + unpricedUsagePolicy: row.unpricedUsagePolicy as "block" | "allow", + metric: row.metric as BudgetMetric, + windowKind: row.windowKind as BudgetWindowKind, + })); + }, + + upsertPolicy: async ( + companyId: string, + input: BudgetPolicyUpsertInput, + actorUserId: string | null, + ): Promise => { + const scope = await resolveScopeRecord(db, input.scopeType, input.scopeId); + if (scope.companyId !== companyId) { + throw unprocessable("Budget scope does not belong to company"); + } + + if (input.reservationCents !== undefined && compareCents(input.reservationCents, 0) < 0) throw unprocessable("Reservation cannot be negative"); + const metric = input.metric ?? "billed_cents"; + const windowKind = input.windowKind ?? (input.scopeType === "project" ? "lifetime" : "calendar_month_utc"); + const existing = await db + .select() + .from(budgetPolicies) + .where( + and( + eq(budgetPolicies.companyId, companyId), + eq(budgetPolicies.scopeType, input.scopeType), + eq(budgetPolicies.scopeId, input.scopeId), + eq(budgetPolicies.metric, metric), + eq(budgetPolicies.windowKind, windowKind), + ), + ) + .then((rows) => rows[0] ?? null); + + if (!existing && input.amount === undefined) throw unprocessable("Amount is required for a new budget policy"); + const amount = Math.max(0, Math.floor(input.amount ?? existing!.amount)); + // A saved zero disables enforcement. Raising it is an explicit limit + // setting; editing a disabled positive policy must preserve that choice. + const nextIsActive = amount > 0 && (input.isActive ?? (existing?.amount === 0 ? true : existing?.isActive) ?? true); + const now = new Date(); + await notifyDeliveryWork(db, DELIVERY_QUEUES.budgetEnforcement, companyId); + const row = existing + ? await db + .update(budgetPolicies) + .set({ + amount, + reservationCents: normalizeCents(input.reservationCents ?? existing.reservationCents ?? 0), + warnPercent: input.warnPercent ?? existing.warnPercent, + hardStopEnabled: input.hardStopEnabled ?? existing.hardStopEnabled, + notifyEnabled: input.notifyEnabled ?? existing.notifyEnabled, + unpricedUsagePolicy: input.unpricedUsagePolicy ?? existing.unpricedUsagePolicy, + isActive: nextIsActive, + updatedByUserId: actorUserId, + updatedAt: now, + enforcementVersion: sql`${budgetPolicies.enforcementVersion} + 1`, + }) + .where(eq(budgetPolicies.id, existing.id)) + .returning() + .then((rows) => rows[0]) + : await db + .insert(budgetPolicies) + .values({ + companyId, + scopeType: input.scopeType, + scopeId: input.scopeId, + metric, + windowKind, + amount, + reservationCents: normalizeCents(input.reservationCents ?? 0), + warnPercent: input.warnPercent ?? 80, + hardStopEnabled: input.hardStopEnabled ?? true, + notifyEnabled: input.notifyEnabled ?? true, + unpricedUsagePolicy: input.unpricedUsagePolicy ?? "block", + isActive: nextIsActive, + createdByUserId: actorUserId, + updatedByUserId: actorUserId, + }) + .returning() + .then((rows) => rows[0]); + + if (input.scopeType === "company" && windowKind === "calendar_month_utc") { + await db + .update(companies) + .set({ + budgetMonthlyCents: amount, + updatedAt: now, + }) + .where(eq(companies.id, input.scopeId)); + } + + if (input.scopeType === "agent" && windowKind === "calendar_month_utc") { + await db + .update(agents) + .set({ + budgetMonthlyCents: amount, + updatedAt: now, + }) + .where(eq(agents.id, input.scopeId)); + } + + if (row.isActive && amount > 0) { + const observed = await computeObservedSpend(db, row); + if (!observedBlocks(row, observed)) { + await resumeScopeFromBudget(row); + await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service", "hard"); + if (row.notifyEnabled && compareCents(observed.totalExact, Math.ceil(row.amount * row.warnPercent / 100)) >= 0) { + await createIncidentIfNeeded(row, "soft", observed); + } else { + await resolveOpenIncidentsForPolicy(row.id, null, null, "soft"); + } + } else { + const softThreshold = Math.ceil((row.amount * row.warnPercent) / 100); + if (row.notifyEnabled && compareCents(observed.totalExact, softThreshold) >= 0 && (!row.hardStopEnabled || compareCents(observed.totalExact, row.amount) < 0)) { + await createIncidentIfNeeded(row, "soft", observed); + } + if (observedBlocks(row, observed)) { + await resolveOpenSoftIncidents(row.id); + await createIncidentIfNeeded(row, "hard", observed); + await pauseAndCancelScopeForBudget(row); + } + } + } else { + await resumeScopeFromBudget(row); + await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service"); + } + + await recordActivity({ + companyId, + actorType: "user", + actorId: actorUserId ?? "board", + action: "budget.policy_upserted", + entityType: "budget_policy", + entityId: row.id, + details: { + scopeType: row.scopeType, + scopeId: row.scopeId, + amount: row.amount, + windowKind: row.windowKind, + }, + }); + + return buildPolicySummary(row); + }, + + overview: async (companyId: string): Promise => { + const rows = await listPolicyRows(companyId); + const policies = await mapBudgetReads(rows, buildPolicySummary); + const activeIncidentRows = await db + .select() + .from(budgetIncidents) + .where(and(eq(budgetIncidents.companyId, companyId), eq(budgetIncidents.status, "open"))) + .orderBy(desc(budgetIncidents.createdAt)); + const activeIncidents = await hydrateIncidentRows(activeIncidentRows); + return { + companyId, + policies, + activeIncidents, + pausedAgentCount: new Set(policies.filter((policy) => policy.scopeType === "agent" && policy.paused).map((policy) => policy.scopeId)).size, + pausedProjectCount: new Set(policies.filter((policy) => policy.scopeType === "project" && policy.paused).map((policy) => policy.scopeId)).size, + pendingApprovalCount: activeIncidents.filter((incident) => incident.approvalStatus === "pending").length, + }; + }, + + evaluateAffectedScopes, + evaluateCostEvent: (event: typeof costEvents.$inferSelect) => evaluateAffectedScopes(event.companyId, event.agentId, event.projectId), + + evaluateAdmission, + getInvocationBlock: async (companyId: string, agentId: string | null, context?: { issueId?: string | null; projectId?: string | null }) => + (await evaluateAdmission(companyId, agentId, context)).block, + + resolveIncident: async ( + companyId: string, + incidentId: string, + input: BudgetIncidentResolutionInput, + actorUserId: string, + ): Promise => { + const incident = await db + .select() + .from(budgetIncidents) + .where(eq(budgetIncidents.id, incidentId)) + .then((rows) => rows[0] ?? null); + if (!incident) throw notFound("Budget incident not found"); + if (incident.companyId !== companyId) throw notFound("Budget incident not found"); + + const policy = await getPolicyRow(incident.policyId); + if (input.action === "raise_budget_and_resume") { + const nextAmount = Math.max(0, Math.floor(input.amount ?? 0)); + const observed = await computeObservedSpend(db, policy); + if (unpricedUsageBlocks(policy, observed)) { + throw unprocessable("Resolve unpriced usage or explicitly allow it in the budget policy before resuming"); + } + if (observed.pendingRunCount > 0) throw unprocessable("Completed runs must finish accounting before resuming"); + if (compareCents(observed.totalExact, nextAmount) >= 0) { + throw unprocessable("New budget must exceed current observed spend"); + } + + const now = new Date(); + await notifyDeliveryWork(db, DELIVERY_QUEUES.budgetEnforcement, companyId); + await db + .update(budgetPolicies) + .set({ + amount: nextAmount, + isActive: true, + updatedByUserId: actorUserId, + updatedAt: now, + enforcementVersion: sql`${budgetPolicies.enforcementVersion} + 1`, + }) + .where(eq(budgetPolicies.id, policy.id)); + + if (policy.scopeType === "company" && policy.windowKind === "calendar_month_utc") { + await db + .update(companies) + .set({ budgetMonthlyCents: nextAmount, updatedAt: now }) + .where(eq(companies.id, policy.scopeId)); + } + + if (policy.scopeType === "agent" && policy.windowKind === "calendar_month_utc") { + await db + .update(agents) + .set({ budgetMonthlyCents: nextAmount, updatedAt: now }) + .where(eq(agents.id, policy.scopeId)); + } + + await resumeScopeFromBudget(policy); + await db + .update(budgetIncidents) + .set({ + status: "resolved", + resolvedAt: now, + updatedAt: now, + }) + .where(and(eq(budgetIncidents.policyId, policy.id), eq(budgetIncidents.status, "open"))); + + await markApprovalStatus(db, incident.approvalId ?? null, "approved", input.decisionNote, actorUserId); + } else { + await db + .update(budgetIncidents) + .set({ + status: "dismissed", + resolvedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(budgetIncidents.id, incident.id)); + await markApprovalStatus(db, incident.approvalId ?? null, "rejected", input.decisionNote, actorUserId); + } + + await recordActivity({ + companyId: incident.companyId, + actorType: "user", + actorId: actorUserId, + action: "budget.incident_resolved", + entityType: "budget_incident", + entityId: incident.id, + details: { + action: input.action, + amount: input.amount ?? null, + scopeType: incident.scopeType, + scopeId: incident.scopeId, + }, + }); + + const [updated] = await hydrateIncidentRows([{ + ...incident, + status: input.action === "raise_budget_and_resume" ? "resolved" : "dismissed", + resolvedAt: new Date(), + updatedAt: new Date(), + }]); + return updated!; + }, + }; +} + +/** Fence the durable stop intent against policy edits and admission. External + * process shutdown must happen after this transaction, without holding locks + * that the executor needs to persist its final usage receipt. */ +export async function withCurrentBudgetEnforcement(db: Db, scope: BudgetEnforcementScope, work: (tx: Db) => Promise): Promise { + return withAccountingTransaction(db, scope.companyId, async (tx) => { + if (scope.enforcement) { + const [policy] = await tx.select().from(budgetPolicies).where(and( + eq(budgetPolicies.id, scope.enforcement.policyId), eq(budgetPolicies.companyId, scope.companyId), + )); + if (!policy || policy.scopeType !== scope.scopeType || policy.scopeId !== scope.scopeId + || policy.enforcementVersion !== scope.enforcement.version || !(await policyBlocks(tx, policy))) return null; + } + return work(tx); + }); +} + +async function deliverScopes(items: T[], deliver: (item: T) => Promise) { + let next = 0; + await Promise.all(Array.from({ length: Math.min(4, items.length) }, async () => { + while (next < items.length) await deliver(items[next++]); + })); +} + +/** Cancellation is an at-least-once external effect. Only the delivery worker + * invokes this: request paths commit intent and return without awaiting it. */ +export async function deliverBudgetEnforcement(db: Db, hooks: BudgetServiceHooks, companyId?: string, _agentId?: string | null) { + if (!hooks.cancelWorkForScope) return; + const attempted = new Set(); + const scopeKey = (scope: BudgetEnforcementScope) => `${scope.companyId}:${scope.scopeType}:${scope.scopeId}`; + const pending = await db.select().from(budgetPolicies).where(and( + sql`${budgetPolicies.enforcementVersion} > ${budgetPolicies.enforcementDeliveredVersion}`, + companyId ? eq(budgetPolicies.companyId, companyId) : undefined, + )).orderBy(budgetPolicies.updatedAt, budgetPolicies.id).limit(100); + await deliverScopes(pending, policy => withBudgetSpan("budget.enforcement.scope", async () => { + try { + const cancellation = await withAccountingTransaction(db, policy.companyId, async tx => { + const [current] = await tx.select().from(budgetPolicies).where(eq(budgetPolicies.id, policy.id)); + if (!current || current.enforcementVersion !== policy.enforcementVersion || !(await policyBlocks(tx, current))) return null; + return { companyId: policy.companyId, scopeType: policy.scopeType as BudgetScopeType, scopeId: policy.scopeId, + createdBefore: new Date(), enforcement: { policyId: policy.id, version: policy.enforcementVersion } }; + }); + if (cancellation) { + attempted.add(scopeKey(cancellation)); + await hooks.cancelWorkForScope!(cancellation); + } + await db.update(budgetPolicies).set({ enforcementDeliveredVersion: policy.enforcementVersion }) + .where(and(eq(budgetPolicies.id, policy.id), eq(budgetPolicies.enforcementVersion, policy.enforcementVersion))); + } catch (error) { + logger.warn({ err: error, policyId: policy.id }, "Budget cancellation delivery pending; accounting is committed"); + // Rotate a failed scope behind other pending work. A poisoned delivery + // must not monopolize the bounded batch on every retry. + await db.update(budgetPolicies).set({ updatedAt: new Date() }).where(and(eq(budgetPolicies.id, policy.id), + eq(budgetPolicies.enforcementVersion, policy.enforcementVersion))).catch(() => undefined); + } + }, { "budget.enforcement.queue_delay_ms": Math.max(0, Date.now() - policy.updatedAt.getTime()) })); + // A committed run Stop outlives a policy edit or deletion. It remains owed + // even if no policy generation now authorizes a fresh cancellation. + const committed = (await listPendingBudgetCancellationScopes(db)) + .filter(scope => (!companyId || scope.companyId === companyId) && !attempted.has(scopeKey(scope))); + await deliverScopes(committed, async scope => { + try { await hooks.cancelWorkForScope!(scope); } + catch (error) { logger.warn({ err: error, companyId: scope.companyId, scopeType: scope.scopeType, scopeId: scope.scopeId }, "Committed budget Stop remains pending"); } + }); +} + +export async function hasPendingBudgetEnforcement(db: Db): Promise { + const [pending] = await db.select({ id: budgetPolicies.id }).from(budgetPolicies) + .where(sql`${budgetPolicies.enforcementVersion} > ${budgetPolicies.enforcementDeliveredVersion}`).limit(1); + return Boolean(pending) || (await listPendingBudgetCancellationScopes(db, 1)).length > 0; +} + +const budgetOwnedScope = or( + and(eq(budgetPolicies.scopeType, "company"), sql`exists (select 1 from ${companies} where ${companies.id} = ${budgetPolicies.scopeId} and ${companies.pauseReason} = 'budget')`), + and(eq(budgetPolicies.scopeType, "agent"), sql`exists (select 1 from ${agents} where ${agents.id} = ${budgetPolicies.scopeId} and ${agents.lifecycleHolds} @> '["budget"]'::jsonb)`), + and(eq(budgetPolicies.scopeType, "project"), sql`exists (select 1 from ${projects} where ${projects.id} = ${budgetPolicies.scopeId} and ${projects.pauseReason} = 'budget')`), +)!; + +const openIncidentScope = sql`exists (select 1 from ${budgetIncidents} + where ${budgetIncidents.policyId} = ${budgetPolicies.id} and ${budgetIncidents.status} = 'open')`; +const unfinishedRunScope = sql`exists (select 1 from ${heartbeatRuns} + where ${heartbeatRuns.companyId} = ${budgetPolicies.companyId} and ${heartbeatRuns.costAccountingPending} = true + and ${heartbeatRuns.costAccountedAt} is null and (${budgetPolicies.scopeType} = 'company' + or (${budgetPolicies.scopeType} = 'agent' and ${heartbeatRuns.agentId} = ${budgetPolicies.scopeId}) + or (${budgetPolicies.scopeType} = 'project' and ${heartbeatRuns.usageJson}->'ledgerScope'->>'projectId' = ${budgetPolicies.scopeId}::text)))`; + +export async function nextBudgetReconciliationAt(db: Db): Promise { + const [held] = await db.select({ id: budgetPolicies.id }).from(budgetPolicies) + .where(and(eq(budgetPolicies.windowKind, "calendar_month_utc"), or(budgetOwnedScope, openIncidentScope))).limit(1); + return held ? currentUtcMonthWindow().end : null; +} + +/** Startup repairs configured policy scopes, not every agent. Afterwards only + * held monthly scopes owe a timed reevaluation; writes reconcile their own scopes. */ +export async function reconcileDueBudgetPolicies(db: Db, options: { startup?: boolean; cursor?: string | null } = {}): Promise<{ nextCursor: string | null }> { + const actionable = options.startup ? or(budgetOwnedScope, openIncidentScope, + sql`${budgetPolicies.enforcementVersion} > ${budgetPolicies.enforcementDeliveredVersion}`, + and(eq(budgetPolicies.isActive, true), eq(budgetPolicies.hardStopEnabled, true), sql`${budgetPolicies.amount} > 0`, unfinishedRunScope)) + : and(eq(budgetPolicies.windowKind, "calendar_month_utc"), or(budgetOwnedScope, openIncidentScope)); + const rows = await db.select({ id: budgetPolicies.id, companyId: budgetPolicies.companyId, scopeType: budgetPolicies.scopeType, scopeId: budgetPolicies.scopeId }) + .from(budgetPolicies).where(and(actionable, options.cursor ? sql`${budgetPolicies.id} > ${options.cursor}::uuid` : undefined)) + .orderBy(budgetPolicies.id).limit(100); + const scopes = new Map(rows.map(scope => [`${scope.companyId}:${scope.scopeType}:${scope.scopeId}`, scope])); + for (const scope of scopes.values()) { + try { + await withAccountingTransaction(db, scope.companyId, (tx, publications) => budgetServiceInTransaction(tx, publications) + .reconcileScope(scope.companyId, scope.scopeType as BudgetScopeType, scope.scopeId)); + } catch (error) { + if (error instanceof HttpError && error.status === 404) continue; + throw error; + } + } + return { nextCursor: rows.length === 100 ? rows[rows.length - 1].id : null }; +} + +export function budgetService(db: Db, _hooks: BudgetServiceHooks = {}) { + const reads = budgetServiceInTransaction(db); + const mutate = (companyId: string, work: (service: ReturnType) => Promise) => + withAccountingTransaction(db, companyId, (tx, publications) => work(budgetServiceInTransaction(tx, publications))); + return { + ...reads, + evaluateAdmission: (companyId: string, agentId: string | null, context?: { issueId?: string | null; projectId?: string | null }) => + mutate(companyId, service => service.evaluateAdmission(companyId, agentId, context)), + evaluateAffectedScopes: (companyId: string, agentId: string | null, projectId: string | null) => + mutate(companyId, service => service.evaluateAffectedScopes(companyId, agentId, projectId)), + reconcileScope: (companyId: string, scopeType: BudgetScopeType, scopeId: string) => + mutate(companyId, service => service.reconcileScope(companyId, scopeType, scopeId)), + reconcilePolicies: () => reconcileDueBudgetPolicies(db, { startup: true }), + upsertPolicy: (companyId: string, input: BudgetPolicyUpsertInput, actorUserId: string | null) => + mutate(companyId, service => service.upsertPolicy(companyId, input, actorUserId)), + evaluateCostEvent: (event: typeof costEvents.$inferSelect) => mutate(event.companyId, service => service.evaluateCostEvent(event)), + getInvocationBlock: (companyId: string, agentId: string | null, context?: { issueId?: string | null; projectId?: string | null }) => + mutate(companyId, service => service.getInvocationBlock(companyId, agentId, context)), + resolveIncident: (companyId: string, incidentId: string, input: BudgetIncidentResolutionInput, actorUserId: string) => + mutate(companyId, service => service.resolveIncident(companyId, incidentId, input, actorUserId)), + }; +} diff --git a/server/src/services/companies.ts b/server/src/services/companies.ts index 0eff1c83613..0292dfa9378 100644 --- a/server/src/services/companies.ts +++ b/server/src/services/companies.ts @@ -1 +1,562 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js"; +import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +import { isRunnerApiResponseAsset } from "./asset-content-access.js"; +import { deleteCompany } from "./company-deletion.js"; +import { fenceActiveCompanyAgentStopsInTransaction } from "./agent-lifecycle.js"; +import { publishAccountingActivities } from "./accounting-transaction.js"; +import { budgetServiceInTransaction, type BudgetServiceHooks } from "./budgets.js"; +import { and, count, eq, gte, inArray, isNull, lt, notInArray, sql } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { + companies, + companyLogos, + assets, + agents, + agentWakeupRequests, + issues, + heartbeatRuns, + costEvents, +} from "@paperclipai/db"; +import { notFound, unprocessable } from "../errors.js"; +import { isCloudManagedInstance } from "./cloud-instance.js"; +import { notifyCloudOfPrimaryCompanyLifecycleChange } from "./cloud-lifecycle-sync.js"; +import { + MAX_ISSUE_PREFIX_ATTEMPTS, + deriveIssuePrefixBase, + isIssuePrefixConflict, + issuePrefixSuffixForAttempt, + pickAvailableIssuePrefix, + rekeyCompanyIssueIdentifiers, +} from "./issue-prefix.js"; +import { environmentService } from "./environments.js"; +import { logActivity, type ActivityPublication } from "./activity-log.js"; +import { builtInAgentService } from "./built-in-agents.js"; + + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +export interface CompanyActivityActor { + actorType: "user" | "agent" | "system" | "plugin"; + actorId: string; + agentId?: string | null; + runId?: string | null; +} + +const SYSTEM_COMPANY_ACTOR: CompanyActivityActor = { + actorType: "system", + actorId: "system", + agentId: null, + runId: null, +}; + +export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { + const environmentsSvc = environmentService(db); + const builtInAgents = builtInAgentService(db); + + type CompanyTx = Parameters[0]>[0]; + + async function applyArchiveCascadeInTx(tx: CompanyTx, id: string) { + const [agentCount] = await tx.select({ value: count() }).from(agents).where(and(eq(agents.companyId, id), notInArray(agents.status, ["paused", "terminated", "pending_approval"]))); + + const [runCount] = await tx + .select({ value: count() }) + .from(heartbeatRuns) + .where(and( + eq(heartbeatRuns.companyId, id), + inArray(heartbeatRuns.status, ["queued", "running", "scheduled_retry"]), + )); + + await notifyChatInboundWakeNotices(tx, { companyId: id }); + await tx + .update(agentWakeupRequests) + .set({ + status: "cancelled", + error: "Cancelled because the company was archived", + finishedAt: new Date(), + updatedAt: new Date(), + }) + .where(and( + eq(agentWakeupRequests.companyId, id), + inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution", "claimed"]), + isNull(agentWakeupRequests.runId), + )); + + return { agentsPauseRequested: agentCount.value, runsCancellationRequested: runCount.value }; + } + + async function finalizeArchive( + id: string, + actor: CompanyActivityActor, + cascade: { agentsPauseRequested: number; runsCancellationRequested: number }, + ) { + // The committed company status blocks dispatch immediately. Lifecycle + // projection owns provider termination, including paused agents with work. + await logActivity(db, { + companyId: id, + actorType: actor.actorType, + actorId: actor.actorId, + agentId: actor.agentId ?? null, + runId: actor.runId ?? null, + action: "company.archived", + entityType: "company", + entityId: id, + details: { + agentsPauseRequested: cascade.agentsPauseRequested, + runsCancellationRequested: cascade.runsCancellationRequested, + }, + }); + } + + const companySelection = { + id: companies.id, + name: companies.name, + description: companies.description, + status: companies.status, + issuePrefix: companies.issuePrefix, + issueCounter: companies.issueCounter, + budgetMonthlyCents: companies.budgetMonthlyCents, + spentMonthlyCents: companies.spentMonthlyCents, + defaultResponsibleUserId: companies.defaultResponsibleUserId, + requireBoardApprovalForNewAgents: companies.requireBoardApprovalForNewAgents, + interactionResolverGovernance: companies.interactionResolverGovernance, + feedbackDataSharingEnabled: companies.feedbackDataSharingEnabled, + feedbackDataSharingConsentAt: companies.feedbackDataSharingConsentAt, + feedbackDataSharingConsentByUserId: companies.feedbackDataSharingConsentByUserId, + feedbackDataSharingTermsVersion: companies.feedbackDataSharingTermsVersion, + logoAssetId: companyLogos.assetId, + createdAt: companies.createdAt, + updatedAt: companies.updatedAt, + }; + + function enrichCompany(company: T) { + return { + ...company, + logoUrl: company.logoAssetId ? `/api/assets/${company.logoAssetId}/content` : null, + }; + } + + function currentUtcMonthWindow(now = new Date()) { + const year = now.getUTCFullYear(); + const month = now.getUTCMonth(); + return { + start: new Date(Date.UTC(year, month, 1, 0, 0, 0, 0)), + end: new Date(Date.UTC(year, month + 1, 1, 0, 0, 0, 0)), + }; + } + + async function getMonthlySpendByCompanyIds( + companyIds: string[], + database: Pick = db, + ) { + if (companyIds.length === 0) return new Map(); + const { start, end } = currentUtcMonthWindow(); + const rows = await database + .select({ + companyId: costEvents.companyId, + spentMonthlyCents: sql`coalesce(sum(${costEvents.costCents}), 0)::double precision`, + }) + .from(costEvents) + .where( + and( + inArray(costEvents.companyId, companyIds), + gte(costEvents.occurredAt, start), + lt(costEvents.occurredAt, end), + ), + ) + .groupBy(costEvents.companyId); + return new Map(rows.map((row) => [row.companyId, Number(row.spentMonthlyCents ?? 0)])); + } + + async function hydrateCompanySpend( + rows: T[], + database: Pick = db, + ) { + const spendByCompanyId = await getMonthlySpendByCompanyIds(rows.map((row) => row.id), database); + return rows.map((row) => ({ + ...row, + spentMonthlyCents: spendByCompanyId.get(row.id) ?? 0, + })); + } + + function getCompanyQuery(database: Pick) { + return database + .select(companySelection) + .from(companies) + .leftJoin(companyLogos, eq(companyLogos.companyId, companies.id)); + } + + /** + * Decides whether a rename must move the company onto a new issue prefix, and + * returns the exact prefix pair to re-key. + * + * Self-hosted companies pick their prefix from the name at creation and keep + * it, so a rename leaves the prefix alone. On a hosted/managed instance the + * company is provisioned for the operator, so the name is the only prefix + * source the operator ever chose — a rename re-derives it. Returns null when + * the current prefix is already correct or when the suffix space is + * exhausted. + */ + async function resolveRenamedIssuePrefix( + tx: CompanyTx, + companyId: string, + companyPatch: Partial, + ): Promise<{ fromPrefix: string; toPrefix: string } | null> { + // Only patch and environment facts gate the lock. Every comparison against + // the company's own name or prefix happens below, under the lock. + // An explicit prefix in the patch is the caller's decision; never override it. + if (companyPatch.issuePrefix !== undefined) return null; + const nextName = companyPatch.name; + if (typeof nextName !== "string" || nextName.trim().length === 0) return null; + if (!isCloudManagedInstance()) return null; + + // Lock the company row before comparing anything against it. Two concurrent + // updates would otherwise each decide from the row they read before either + // committed, and both ways of getting that wrong end with a company whose + // prefix disagrees with its own identifiers: + // + // - Two renames: the second re-keys from the prefix it read, finds the + // identifiers the first already moved, and leaves them on the first + // rename's prefix while the row carries the second one's. + // - A rename plus a stale form that resubmits the original name: the + // second sees a name equal to the one it read, skips re-derivation, and + // restores the old name on top of the first rename's prefix. + // + // Reading the row under the lock makes the second transaction decide from + // what the first actually committed. Only a managed instance takes this + // lock, and only for an update that carries a name. + const locked = await tx + .select({ name: companies.name, issuePrefix: companies.issuePrefix }) + .from(companies) + .where(eq(companies.id, companyId)) + .for("update") + .then((rows) => rows[0] ?? null); + if (!locked || nextName === locked.name) return null; + + const nextBase = deriveIssuePrefixBase(nextName); + // A rename that keeps the same base keeps the current prefix, including + // any disambiguating suffix it was allocated. + if (nextBase === deriveIssuePrefixBase(locked.name)) return null; + if (nextBase === locked.issuePrefix) return null; + + const candidate = await pickAvailableIssuePrefix(tx, nextBase); + if (!candidate || candidate === locked.issuePrefix) return null; + return { fromPrefix: locked.issuePrefix, toPrefix: candidate }; + } + + async function createCompanyWithUniquePrefix(data: typeof companies.$inferInsert) { + const base = deriveIssuePrefixBase(data.name); + let suffix = 1; + while (suffix <= MAX_ISSUE_PREFIX_ATTEMPTS) { + const candidate = `${base}${issuePrefixSuffixForAttempt(suffix)}`; + try { + const rows = await db + .insert(companies) + .values({ ...data, issuePrefix: candidate }) + .returning(); + return rows[0]; + } catch (error) { + if (!isIssuePrefixConflict(error)) throw error; + } + suffix += 1; + } + throw new Error("Unable to allocate unique issue prefix"); + } + + return { + list: async () => { + const rows = await getCompanyQuery(db); + const hydrated = await hydrateCompanySpend(rows); + return hydrated.map((row) => enrichCompany(row)); + }, + + getById: async (id: string) => { + // Non-UUID refs previously reached the uuid-typed query and threw a + // DrizzleQueryError ("invalid input syntax for type uuid"), surfacing + // as HTTP 500 from GET /api/companies/:companyId. Treat them as + // not-found so the route returns 404. + if (!UUID_RE.test(id)) return null; + const row = await getCompanyQuery(db) + .where(eq(companies.id, id)) + .then((rows) => rows[0] ?? null); + if (!row) return null; + const [hydrated] = await hydrateCompanySpend([row], db); + return enrichCompany(hydrated); + }, + + create: async (data: typeof companies.$inferInsert) => { + const created = await createCompanyWithUniquePrefix(data); + await environmentsSvc.ensureLocalEnvironment(created.id); + await builtInAgents.autoProvisionBundledAgents(created.id); + const row = await getCompanyQuery(db) + .where(eq(companies.id, created.id)) + .then((rows) => rows[0] ?? null); + if (!row) throw notFound("Company not found after creation"); + const [hydrated] = await hydrateCompanySpend([row], db); + return enrichCompany(hydrated); + }, + + update: async ( + id: string, + data: Partial & { logoAssetId?: string | null }, + actor: CompanyActivityActor = SYSTEM_COMPANY_ACTOR, + ) => { + const budgetPublications: ActivityPublication[] = []; + const result = await db.transaction(async (tx) => { + if (data.status !== undefined || data.budgetMonthlyCents !== undefined) { + await tx.select({ id: companies.id }).from(companies).where(eq(companies.id, id)).for("no key update"); + } + const existing = await getCompanyQuery(tx) + .where(eq(companies.id, id)) + .then((rows) => rows[0] ?? null); + if (!existing) return null; + + const { logoAssetId, ...companyPatch } = data; + const willReactivate = existing.status !== "active" && companyPatch.status === "active"; + const willArchive = existing.status !== "archived" && companyPatch.status === "archived"; + + if (logoAssetId !== undefined && logoAssetId !== null) { + const nextLogoAsset = await tx + .select() + .from(assets) + .where(eq(assets.id, logoAssetId)) + .then((rows) => rows[0] ?? null); + if (!nextLogoAsset) throw notFound("Logo asset not found"); + if (await isRunnerApiResponseAsset(tx, nextLogoAsset)) { + throw unprocessable("API response snapshots cannot be company logos; upload a logo asset"); + } + if (nextLogoAsset.companyId !== existing.id) { + throw unprocessable("Logo asset must belong to the same company"); + } + } + + const renamedPrefix = await resolveRenamedIssuePrefix(tx, id, companyPatch); + + if (companyPatch.status !== undefined && companyPatch.status !== existing.status) { + await notifyRunDispatch(tx, { companyId: id }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, id); + await notifyCompanyRetentionWork(tx, id); + await notifyChatInboundWakeNotices(tx, { companyId: id }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id); + } + const updated = await tx + .update(companies) + .set({ + ...companyPatch, + ...(renamedPrefix ? { issuePrefix: renamedPrefix.toPrefix } : {}), + updatedAt: new Date(), + }) + .where(eq(companies.id, id)) + .returning() + .then((rows) => rows[0] ?? null); + if (!updated) return null; + + let issuePrefixRederived: { + previousIssuePrefix: string; + issuePrefix: string; + issuesRekeyed: number; + casesRekeyed: number; + } | null = null; + if (renamedPrefix) { + const rekeyed = await rekeyCompanyIssueIdentifiers(tx, { + companyId: id, + fromPrefix: renamedPrefix.fromPrefix, + toPrefix: renamedPrefix.toPrefix, + }); + issuePrefixRederived = { + previousIssuePrefix: renamedPrefix.fromPrefix, + issuePrefix: renamedPrefix.toPrefix, + issuesRekeyed: rekeyed.issues, + casesRekeyed: rekeyed.cases, + }; + } + + let agentsRestored = 0; + if (willReactivate) { + const restoredRows = await tx.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, id), eq(agents.pauseReason, "company_archived"))); + agentsRestored = restoredRows.length; + } + + const archiveCascade = willArchive ? await applyArchiveCascadeInTx(tx, id) : null; + if (companyPatch.status !== existing.status && (companyPatch.status === "paused" || companyPatch.status === "archived")) { + await fenceActiveCompanyAgentStopsInTransaction(tx as unknown as Db, id, companyPatch.status); + } + + if (logoAssetId === null) { + await tx.delete(companyLogos).where(eq(companyLogos.companyId, id)); + } else if (logoAssetId !== undefined) { + await tx + .insert(companyLogos) + .values({ + companyId: id, + assetId: logoAssetId, + }) + .onConflictDoUpdate({ + target: companyLogos.companyId, + set: { + assetId: logoAssetId, + updatedAt: new Date(), + }, + }); + } + + if (logoAssetId !== undefined && existing.logoAssetId && existing.logoAssetId !== logoAssetId) { + await tx.delete(assets).where(eq(assets.id, existing.logoAssetId)); + } + + if (data.budgetMonthlyCents !== undefined) { + await budgetServiceInTransaction(tx as unknown as Db, budgetPublications).upsertPolicy(id, { + scopeType: "company", scopeId: id, amount: data.budgetMonthlyCents, isActive: data.budgetMonthlyCents > 0, windowKind: "calendar_month_utc", + }, actor.actorType === "user" ? actor.actorId : null); + const [budgetUpdated] = await tx.select().from(companies).where(eq(companies.id, id)); + Object.assign(updated, budgetUpdated); + } + const [hydrated] = await hydrateCompanySpend([{ + ...updated, + logoAssetId: logoAssetId === undefined ? existing.logoAssetId : logoAssetId, + }], tx); + + const shouldLogReactivation = willReactivate && + (existing.status === "archived" || agentsRestored > 0); + + return { + company: enrichCompany(hydrated), + reactivated: shouldLogReactivation ? { agentsRestored } : null, + archiveCascade, + unarchived: willReactivate && existing.status === "archived", + issuePrefixRederived, + }; + }); + if (!result) return null; + publishAccountingActivities(id, budgetPublications); + // Post-commit, fire-and-forget, and BEFORE any finalization that + // could throw: a Cloud-pinned primary company that crossed the + // archived boundary (either direction) rings the harness so the + // stack itself can converge. The status transaction has already + // committed, so a later cascade or activity-log failure must not + // leave Cloud unaware of a company that is in fact archived. + if (result.archiveCascade || result.unarchived) { + void notifyCloudOfPrimaryCompanyLifecycleChange(id); + } + if (result.issuePrefixRederived) { + await logActivity(db, { + companyId: id, + actorType: actor.actorType, + actorId: actor.actorId, + agentId: actor.agentId ?? null, + runId: actor.runId ?? null, + action: "company.updated", + entityType: "company", + entityId: id, + details: { + source: "company_rename", + reason: "issue_prefix_rederived", + ...result.issuePrefixRederived, + }, + }); + } + if (result.reactivated) { + await logActivity(db, { + companyId: id, + actorType: actor.actorType, + actorId: actor.actorId, + agentId: actor.agentId ?? null, + runId: actor.runId ?? null, + action: "company.reactivated", + entityType: "company", + entityId: id, + details: { agentsRestored: result.reactivated.agentsRestored }, + }); + } + if (result.archiveCascade) { + await finalizeArchive(id, actor, result.archiveCascade); + } + return result.company; + }, + + archive: async (id: string, actor: CompanyActivityActor = SYSTEM_COMPANY_ACTOR) => { + const result = await db.transaction(async (tx) => { + const existing = await tx + .select({ status: companies.status }) + .from(companies) + .where(eq(companies.id, id)) + .for("no key update") + .then((rows) => rows[0] ?? null); + if (!existing) return null; + + const wasAlreadyArchived = existing.status === "archived"; + + if (!wasAlreadyArchived) { + await notifyRunDispatch(tx, { companyId: id }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, id); + await notifyCompanyRetentionWork(tx, id); + await notifyChatInboundWakeNotices(tx, { companyId: id }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id); + await tx + .update(companies) + .set({ status: "archived", updatedAt: new Date() }) + .where(eq(companies.id, id)); + } + + const cascade = wasAlreadyArchived ? null : await applyArchiveCascadeInTx(tx, id); + if (!wasAlreadyArchived) await fenceActiveCompanyAgentStopsInTransaction(tx as unknown as Db, id, "archived"); + + const row = await getCompanyQuery(tx) + .where(eq(companies.id, id)) + .then((rows) => rows[0] ?? null); + if (!row) return null; + const [hydrated] = await hydrateCompanySpend([row], tx); + return { + company: enrichCompany(hydrated), + cascade, + }; + }); + if (!result) return null; + + // Same doorbell rule as update(): the archive is committed, so ring + // before finalization, which can throw without undoing it. + if (result.cascade) { + void notifyCloudOfPrimaryCompanyLifecycleChange(id); + await finalizeArchive(id, actor, result.cascade); + } + + return result.company; + }, + + remove: (id: string) => deleteCompany(db, id), + + stats: () => + Promise.all([ + db + .select({ companyId: agents.companyId, count: count() }) + .from(agents) + .groupBy(agents.companyId), + db + .select({ companyId: issues.companyId, count: count() }) + .from(issues) + .groupBy(issues.companyId), + ]).then(([agentRows, issueRows]) => { + const result: Record = {}; + for (const row of agentRows) { + result[row.companyId] = { agentCount: row.count, issueCount: 0 }; + } + for (const row of issueRows) { + if (result[row.companyId]) { + result[row.companyId].issueCount = row.count; + } else { + result[row.companyId] = { agentCount: 0, issueCount: row.count }; + } + } + return result; + }), + }; +} diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 6b39445d0ba..c1593839a35 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -2,4 +2,136 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; /** Topics for existing durable queues; register intent before their writes. */ export const DELIVERY_QUEUES = { + scheduledRetryDispatch: "scheduled-retry-dispatch", + queuedRunDispatch: "queued-run-dispatch", statusCardEvaluation: "status-card-evaluation", + runEvidenceMigration: "run-evidence-migration", + providerTraceRetention: "provider-trace-retention", + nativeRunnerEvidenceRetention: "native-runner-evidence-retention", + skillVersionRetention: "skill-version-retention", + databaseBackups: "database-backups", + importSpoolCleanup: "import-spool-cleanup", + externalConversationSettlement: "external-conversation-settlement", + nativeRecoveryNotices: "native-recovery-notices", + nativeEnvironmentSettlement: "native-environment-settlement", + nativeRunProjection: "native-run-projection", + nativeResultCommit: "native-result-commit", + nativeWorkspaceFinalization: "native-workspace-finalization", + retainedNativeSessionCleanup: "retained-native-session-cleanup", + nativeStatusReassessment: "native-status-reassessment", + nativeReviewResponse: "native-review-response", + nativeResponsePresentation: "native-response-presentation", + nativePendingEffects: "native-pending-effects", + unsafeWorkspaceExportRecovery: "unsafe-workspace-export-recovery", + automaticCompletionReviews: "automatic-completion-reviews", + obsoletePolicyReviews: "obsolete-policy-reviews", + cancelledWorkspacePreservation: "cancelled-workspace-preservation", + instructionCollection: "instruction-collection", + directoryCleanup: "directory-cleanup", + instructionCommit: "instruction-commit", + scheduledRetryCancellation: "scheduled-retry-cancellation", + executionFinalization: "execution-finalization", + workspaceRecovery: "workspace-recovery", + executionReplacement: "execution-replacement", + executionContinuation: "execution-continuation", + executionStatus: "execution-status", + executionDisposition: "execution-disposition", + environmentCleanup: "environment-cleanup", + approvalQueueSeeding: "approval-queue-seeding", + approvalRetention: "approval-retention", + joinRequestRetention: "join-request-retention", + decisionTargets: "decision-targets", + decisionEffectRecovery: "decision-effect-recovery", + decisionExpiry: "decision-expiry", + decisionContinuation: "decision-continuation", + + decisionArchiveDelivery: "decision-archive-delivery", + toolConnectionHealth: "tool-connection-health", + toolReviewProjection: "tool-review-projection", + toolReviewInterruption: "tool-review-interruption", + toolReviewRecovery: "tool-review-recovery", + toolReviewExpiry: "tool-review-expiry", + gatewayCredentialExpiry: "gateway-credential-expiry", + secretProposalExpiry: "secret-proposal-expiry", + customImageExpiry: "custom-image-expiry", + historicalCleanup: "historical-cleanup", + adapterLoginCleanup: "adapter-login-cleanup", + setupTokenCleanup: "setup-token-cleanup", + localLoginCleanup: "local-login-cleanup", + budgetEnforcement: "budget-enforcement", + accounting: "cost-accounting", + taskDrainExpiry: "task-drain-expiry", + agentLifecycle: "agent-lifecycle", + chatBoardMessages: "chat-board-messages", + chatTaskStarts: "chat-task-starts", + chatReceiptReactions: "chat-receipt-reactions", + chatSessionStops: "chat-session-stops", + chatProviderEffects: "chat-provider-effects", + chatGitHubWebhookRecovery: "chat-github-webhook-recovery", + chatGitHubIngress: "chat-github-ingress", + chatGitHubPublications: "chat-github-publications", + chatGitHubReviewProjection: "chat-github-review-projection", + chatGitHubReviewChecks: "chat-github-review-checks", + chatGitHubTerminalResponses: "chat-github-terminal-responses", + chatGitHubAcknowledgements: "chat-github-acknowledgements", + chatFailedRunRetries: "chat-failed-run-retries", + chatTelegramMaintenance: "chat-telegram-maintenance", + chatReceivers: "chat-receivers", + voiceSessionRecovery: "voice-session-recovery", + voiceReports: "voice-call-reports", + voiceInboundRetention: "voice-inbound-retention", + voiceInboundCleanup: "voice-inbound-cleanup", + voiceReplies: "voice-reply-delivery", + chatTeamsConsentExpiry: "chat-teams-consent-expiry", + chatTeamsAttemptRecovery: "chat-teams-attempt-recovery", + chatCommandRegistration: "chat-command-registration", + chatTelegramSubscriptions: "chat-telegram-subscriptions", + chatSessionSyncs: "chat-session-syncs", + chatVerification: "chat-verification", + chatInbound: "chat-inbound", + chatPublications: "chat-publications", + chatNativeProgress: "chat-native-progress", + chatInboundWakeNotices: "chat-inbound-wake-notices", + chatInteractionContinuations: "chat-interaction-continuations", + chatTerminalConfirmations: "chat-terminal-confirmations", + chatFailedRetryNotices: "chat-failed-retry-notices", + chatRunMilestones: "chat-run-milestones", + chatReceipts: "chat-slack-receipts", + browser: "browser-use", + email: "email-channels", + museMaintenance: "muse-maintenance", + mcpEvents: "mcp-events", + dotEvents: "dot-events", + grokDispatch: "grok-dispatch", + skillSourceSync: "skill-source-sync", + skillPackages: "skill-packages", + runtimeAssetRetention: "runtime-asset-retention", + fastResponse: "fast-responses", + feedback: "feedback-exports", + chatCompletion: "chat-completions", + connection: "connection-continuations", + gitRestorePoints: "git-restore-points", + nativeWorkspaceSyncRetention: "native-workspace-sync-retention", + question: "question-responses", + pluginSchedules: "plugin-schedules", + pluginLifecycle: "plugin-resource-lifecycle", + toolAction: "tool-action-receipts", + terminalWorkspaces: "terminal-workspaces", + routineSchedules: "routine-schedules", + issueMonitors: "issue-monitors", + agentTimers: "agent-timers", + caseHandoffs: "case-handoffs", + nativeCheckpointExpiration: "native-checkpoint-expiration", +} as const; +export type DeliveryQueue = typeof DELIVERY_QUEUES[keyof typeof DELIVERY_QUEUES]; + +export async function notifyDeliveryWork(transaction: object, queue: DeliveryQueue, hint?: string): Promise { + await signalDatabaseWork(transaction, queue, hint); +} +// Public delivery notification observers only see settled work. The coordinator +// uses the lower-level lifecycle subscription to also fence in-flight writes. +export function subscribeDeliveryWork(owner: object, queue: DeliveryQueue, wake: () => void): () => void { + return subscribeDatabaseWork(owner, queue, event => { + if (event === "settled") wake(); + }); +} diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index 01ef4cf933e..84fc80d9ea7 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -1 +1,32 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import type { Db } from "@paperclipai/db"; +import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; + +export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: string; issueId?: string }; + +export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) { + if (scope) { + await notifyRunDispatch(transaction, scope); + await notifyStatusCardIssueChange(transaction, scope.companyId); + } + for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, + DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { + await notifyDeliveryWork(transaction, queue); + } +} + +export function writeExecutionControl( + db: Db | Parameters[0]>[0], + queue: DeliveryQueue | readonly DeliveryQueue[], + write: (tx: Db) => PromiseLike, + scope: ExecutionWorkScope | undefined, +): Promise { + return db.transaction(async transaction => { + await notifyRunDispatch(transaction, scope); if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId); + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); + for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); + return write(transaction as unknown as Db); + }); +} diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 0e6a67a4dcb..16207b0fb28 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -1 +1,5084 @@ +import { notifyRunDispatch } from "../run-dispatch-notifications.js"; import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; +import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; +import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js"; +import { notifyUnsafeWorkspaceExportRecovery, notifyUnsafeWorkspaceExportRunChange } from "../unsafe-workspace-export-notifications.js"; +import { notifyCancelledWorkspacePreservation } from "../cancelled-workspace-preservation-notifications.js"; +import { notifyInstructionCollection } from "../instruction-collection-notifications.js"; +import { notifyScheduledRetryCancellation } from "../scheduled-retry-cancellation-notifications.js"; +import { notifyChatResponseAvailability } from "../chat-response-availability.js"; +import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js"; +import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; +import { assertTaskRetryConversationAllowed, readTaskRetryIntent, readTaskRetryBoardMessageOrigin, retireSupersededTaskRetry, taskRetryMutationAllowed } from "../task-retry.js"; +import { insertIssueComment } from "../issue-comment-writes.js"; +import { withExternalAdmissionGuard, assertNoExternalOverlap } from "../../modules/external-agents/index.js"; +import { isAgentAwaitingSetup } from "../../modules/agent-lifecycle/index.js"; +import { normalizeAgentNameKey } from "./retries.js"; +import { publishActiveDotComment } from "../dot-assignment-follow-up.js"; +import { publishActiveMuseComment } from "../muse-assignment-follow-up.js"; +import { + type WakeupOptions, + mergeCoalescedContextSnapshot, + enrichWakeContextSnapshot, + hasInteractionContinuationWakeContext, + isInteractionResolutionWakePayload, +} from "./run-preparation.js"; +import { resolveExecutionWorkspaceReuseRequestForIssue } from "./workspaces.js"; +import { + CHAT_COMPLETION_WAKE_REASON, + isCompletedOnboardingHandoffWake, +} from "../chat-completion-delivery.js"; +import { + compareCents, + hasWorkspaceRestoreFailure, +} from "@paperclipai/shared"; +import { nonIdleSlackIssueCondition } from "../slack-conversation-state.js"; +import { readQueuedInteractionResponse } from "../queued-interaction-response.js"; +import { + isConversation, + isConversationReset, + isConversationExecutionWake, + isWaitingConversation, +} from "../agent-conversations.js"; +import { legacyControllerClaim } from "../legacy-controller-lease.js"; +import { + admitExplicitNativeContinuation, + undeliveredLegacyUserCommentIds, +} from "../explicit-native-continuation.js"; +import { canRetryStoppedRun } from "../cancelled-native-startup.js"; +import { getExecutionBlocker, hasPendingRunCleanup } from "../execution-blocker.js"; +import { validateNativeWorkspaceReconstruction } from "../native-runtime/native-workspace-reconstruction.js"; +import { isConversationAdapter } from "../conversation-continuation.js"; +import { recordExecutionWait } from "../execution-wait.js"; +import { + getNativeReviewAssignment, + readNativeReviewAssignmentContext, +} from "../native-runtime/native-review-participant.js"; +import { claimQueuedNativeReviewRun } from "../native-runtime/native-review-dispatch.js"; +import { adapterExecutionControls } from "../adapter-execution-control.js"; +import { executionFailureRetryCount } from "../execution-recovery-attempt.js"; +import { + slackMentionAllowsRunStart, + resumeDeferredChatWake, + assertDurableChatWakeupReceipt, + assertDurableChatWakeupRequest, + withDurableChatAdmission, + authorizeFailedChatRunRetryWake, + FailedChatRunRetryAuthorizationError, + unadmittedChatWakeupCondition, +} from "../durable-chat-wakeup.js"; +import { agentService } from "../agents.js"; +import { normalizeLegacyRunnerProvider } from "@paperclipai/adapter-utils"; +import { randomUUID } from "node:crypto"; +import { + and, + asc, + desc, + eq, + gt, + gte, + inArray, + isNull, + lt, + lte, + ne, + notInArray, + or, + sql, +} from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { AGENT_DEFAULT_MAX_CONCURRENT_RUNS } from "@paperclipai/shared"; +import { + agents, + caseHandoffs, + cases, + agentWakeupRequests, + activityLog, + chatConversations, + chatEndpoints, + companies, + costEvents, + executionWorkspaces, + heartbeatRunEvents, + heartbeatRuns, + issueComments, + issueRecoveryActions, + issueRelations, + issues, +} from "@paperclipai/db"; +import { + conflict, + HttpError, + notFound, +} from "../../errors.js"; +import { logger } from "../../middleware/logger.js"; +import { publishLiveEvent } from "../live-events.js"; +import { allocateHeartbeatRunEventSeq } from "../heartbeat-run-events.js"; +import { + queuedCommentIdsFromWakePayload, + queuedCommentIdsFromRunContext, + withQueuedCommentIdsInWakePayload, + withQueuedCommentIdsInRunContext, +} from "../issue-queued-comment-queue.js"; +import { + parseObject, + asBoolean, + asNumber, +} from "../../adapters/utils.js"; +import { isExternalChatWaitAuthorizationContention } from "../native-runtime/chat-attachment-reuse.js"; +import { emitAgentTaskRun } from "../agent-task-run-telemetry.js"; +import { + CHAT_CONTROL_RECOVERY_ADMISSION_KEY, + CHAT_CONTROL_RECOVERY_STOP_CODE, + CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE, + chatControlRecoveryAdmission, + readChatControlRecoveryAdmission, + readChatControlNativeParent, + readChatControlRecoveryStop, +} from "../chat-control-recovery-stop.js"; +import { + ISSUE_NEW_INPUT_ACTIVITY_ACTIONS, + ISSUE_PROGRESS_ACTIVITY_ACTIONS, + ISSUE_REWAKE_LOOKBACK_MS, + ISSUE_REWAKE_RUN_SAMPLE_LIMIT, + evaluateIssueRewakeThrottle, + isThrottleCandidateIssueRewake, +} from "../issue-rewake-throttle.js"; +import { logActivity } from "../activity-log.js"; +import { ISSUE_BLOCKERS_RESOLVED_WAKE_REASON } from "../issue-dependency-wakeups.js"; +import { + ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS, + isVerifiedIssueTreeControlInteractionWake, +} from "../issue-tree-control.js"; +import { + buildExecutionWorkspaceAdapterConfig, + isUnrunnableWorktreeCombo, + parseIssueExecutionWorkspaceSettings, + resolveEffectiveWorkspaceStrategyType, + resolveExecutionWorkspaceMode, + WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, + WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE, + WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION, +} from "../execution-workspace-policy.js"; +import { readContinuationAttempt } from "../recovery/index.js"; +import { allowsIssueInteractionWake, wakeChatConversationsConflict } from "../../modules/run-dispatch/index.js"; +import { withAgentStartLock } from "../agent-start-lock.js"; +import { + evaluateAgentInvokability, + shouldCancelRunsForNonInvokableAgent, + type AgentOrgRow, +} from "../agent-invokability.js"; +import { isHeartbeatWakeOnDemandEnabled, supportsUnscopedTimerHeartbeat } from "../heartbeat-policy.js"; +import { isUuidLike } from "@paperclipai/shared"; +import { retryChatControlAdmission } from "../chat-control-admission-retry.js"; + +import type { HeartbeatServiceOptions } from "../heartbeat.js"; +import type { createHeartbeatRunState } from "./run-state.js"; +import type { createHeartbeatRunPreparation } from "./run-preparation.js"; +import type { HeartbeatRetryDependencies } from "./retries.js"; +import type { HeartbeatRecoveryDependencies, createHeartbeatRecovery } from "./recovery.js"; +import type { issueService } from "../issues.js"; +import type { issueTreeControlService } from "../issue-tree-control.js"; +import type { budgetService } from "../budgets.js"; +import type { instanceSettingsService } from "../instance-settings.js"; +import type { createRunDispatch } from "../../modules/run-dispatch/index.js"; +import type { PostCommitEffect as WakeQueuePostCommitEffect, createWakeQueue } from "../../modules/wake-queue/index.js"; + +type HeartbeatRun = typeof heartbeatRuns.$inferSelect; +type HeartbeatRunState = ReturnType; +type RunPreparation = ReturnType; + +/** Queue policy uses the service's lifecycle effects and process-wide ownership sets. */ +export interface HeartbeatQueueDependencies extends Pick, + Pick, + Pick, + Pick { + sweepPendingCleanupLeases: ReturnType["sweepPendingCleanupLeases"]; + options: Pick; + issuesSvc: Pick, "getDependencyReadiness" | "listDependencyReadiness">; + treeControlSvc: Pick, "getActivePauseHoldGate">; + budgets: Pick, "getInvocationBlock">; + instanceSettings: Pick, "getExperimental">; + runDispatch: Pick, "cancelStaleQueuedRun">; + wakeQueue: Pick, "createAdmissionTransactionScope" | "admitWakeBehindIssueExecution">; + activeRunExecutions: Set; + activeRunExecutionPromises: Set>; + activeWakeupPromises: Set>; + liveRunExecutions: { has(id: string): boolean }; + isHeartbeatRunTerminalStatus: (status: string | null | undefined) => boolean; + isSameTaskScope: (left: string | null, right: string | null) => boolean; + runTaskKey: (run: HeartbeatRun) => string | null; + filterZombieCoalesceTarget: (run: T | null, live: { has(id: string): boolean }) => T | null; + getSchedulingSuppression: () => Promise<{ suppressed: boolean; reason: "worktree_instance" | "database_restore_in_progress" | "task_drain" | null }>; + canStartNewExternalBotWork: (agent: typeof agents.$inferSelect) => Promise; + executeRun: (runId: string) => Promise; + releaseIssueExecutionAndPromote: (run: Pick, options?: { suppressImmediateRecovery?: boolean; resumeStrandedQueue?: boolean; deferredPostCommitEffects?: WakeQueuePostCommitEffect[] }) => Promise; + applyWakeQueuePostCommitEffects: (effects: WakeQueuePostCommitEffect[]) => Promise; + publishRunLifecyclePluginEvent: (run: HeartbeatRun) => void; + cancelRunInternal: (runId: string, reason?: string, options?: { errorCode?: string }) => Promise; + cancelActiveForAgentInternal: (agentId: string, reason: string) => Promise; + resumeExecutionWaitComments: () => Promise; + resumeQueuedCommentInterrupt: (companyId: string, queueId: string) => Promise; + resumeSavedLegacyComments: (companyId: string, queueId: string) => Promise; + formatIssueIdentifierLink: (identifier: string | null, fallback: string) => string; +} + +const HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT = AGENT_DEFAULT_MAX_CONCURRENT_RUNS; + +const HEARTBEAT_MAX_CONCURRENT_RUNS_MIN = 1; + +const HEARTBEAT_MAX_CONCURRENT_RUNS_MAX = 50; + +export const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext"; + +export const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = [ + "queued", + "running", + "scheduled_retry", +] as const; + +const TIMER_ACTIONABLE_ISSUE_STATUSES = ["todo", "in_progress"] as const; + +export class ChatControlRecoveryUnresolvedError extends Error { + constructor() { + super( + "Run admission could not acquire its database locks after bounded retries. No provider work started. Review database contention and send a fresh request; this attempt will not automatically retry.", + ); + } +} + +const RUNNING_ISSUE_WAKE_REASONS_REQUIRING_FOLLOWUP = new Set([ + CHAT_COMPLETION_WAKE_REASON, + "approval_approved", + ISSUE_BLOCKERS_RESOLVED_WAKE_REASON, + "issue_recovery_action_restored", +]); + +function normalizeMaxConcurrentRuns(value: unknown) { + const parsed = Math.floor( + asNumber(value, HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT), + ); + if (!Number.isFinite(parsed)) return HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT; + return Math.max( + HEARTBEAT_MAX_CONCURRENT_RUNS_MIN, + Math.min(HEARTBEAT_MAX_CONCURRENT_RUNS_MAX, parsed), + ); +} + +export function shouldRequireIssueCommentForWake( + contextSnapshot: Record | null | undefined, +) { + if (contextSnapshot?.skipIssueComment === true) return false; + + const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason); + return ( + wakeReason === "issue_assigned" || + wakeReason === "execution_review_requested" || + wakeReason === "execution_approval_requested" || + wakeReason === "execution_changes_requested" + ); +} + +async function listUnresolvedBlockerSummaries( + dbOrTx: Pick, + companyId: string, + issueId: string, + unresolvedBlockerIssueIds: string[], +) { + const ids = [...new Set(unresolvedBlockerIssueIds.filter(Boolean))]; + if (ids.length === 0) return []; + return dbOrTx + .select({ + id: issues.id, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + priority: issues.priority, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.type, "blocks"), + eq(issueRelations.relatedIssueId, issueId), + inArray(issues.id, ids), + ), + ) + .orderBy(asc(issues.title)); +} + +export function shouldDeferFollowupWakeForSameIssue(input: { + activeRunStatus: string | null | undefined; + isSameExecutionAgent: boolean; + wakeCommentId: string | null | undefined; + forceFreshSession: boolean; +}) { + // A comment follow-up or explicit fresh-session wake needs a new run boundary. + if (!input.isSameExecutionAgent) return false; + if (input.activeRunStatus !== "running") return false; + if (input.wakeCommentId) return true; + if (input.forceFreshSession) return true; + return false; +} + +export function shouldQueueFollowupForRunningIssueWake(input: { + contextSnapshot: Record | null | undefined; + wakeCommentId: string | null; +}) { + if (input.wakeCommentId) return true; + // A structured interaction response is new input just like a comment. It + // must run after the turn that created the interaction instead of being + // merged into that still-running turn. + if ( + readNonEmptyString(input.contextSnapshot?.interactionId) && + readNonEmptyString(input.contextSnapshot?.interactionStatus) + ) { + return true; + } + const wakeReason = readNonEmptyString(input.contextSnapshot?.wakeReason); + if (wakeReason === "issue_children_completed" && ( + input.contextSnapshot?.onboardingCompletion === true || + (input.contextSnapshot?.statusDecisionSource === "native_status_decision" && + readNonEmptyString(input.contextSnapshot?.nativeChildCompletionDecisionId)) + )) return true; + return Boolean( + wakeReason && RUNNING_ISSUE_WAKE_REASONS_REQUIRING_FOLLOWUP.has(wakeReason), + ); +} + +function readNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +/** Queue admission and dispatch share the service lifecycle callbacks and execution ownership. */ +export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDependencies) { + const { + setRunStatus, + setWakeupStatus, + appendRunEvent, + releaseIssueExecutionAndPromote, + applyWakeQueuePostCommitEffects, + issuesSvc, + options, + publishRunLifecyclePluginEvent, + finalizeAgentStatus, + getAgent, + cancelRunInternal, + toAgentOrgRow, + getAgentInvokability, + budgets, + activeRunExecutions, + treeControlSvc, + runDispatch, + applyRunDispatchPostCommitEffects, + resolveResponsibleUserIdForRun, + getIssueExecutionContext, + isHeartbeatRunTerminalStatus, + liveRunExecutions, + getSchedulingSuppression, + resumeExecutionWaitComments, + getWorktreeExecutionCutoff, + resumeQueuedCommentInterrupt, + resumeSavedLegacyComments, + cancelActiveForAgentInternal, + listCompanyAgentOrgRows, + executeRun, + canStartNewExternalBotWork, + activeRunExecutionPromises, + activeWakeupPromises, + instanceSettings, + getRun, + sweepPendingCleanupLeases, + resolveExplicitResumeSessionOverride, + resolveSessionBeforeForWakeup, + hasResolvablePriorSessionWorkspaceForWake, + getRoutineEnvForExecutionIssue, + resolveResponsibleUserIdForRunSeed, + formatIssueIdentifierLink, + wakeQueue, + isSameTaskScope, + runTaskKey, + filterZombieCoalesceTarget, + } = dependencies; + + function parseHeartbeatPolicy(agent: typeof agents.$inferSelect) { + const runtimeConfig = parseObject(agent.runtimeConfig); + const heartbeat = parseObject(runtimeConfig.heartbeat); + + return { + enabled: asBoolean(heartbeat.enabled, false), + intervalSec: Math.max(0, asNumber(heartbeat.intervalSec, 0)), + wakeOnDemand: isHeartbeatWakeOnDemandEnabled(agent), + // A Dot binding has one external turn. Competing assignments must retain + // their queue position instead of claiming a second run that cannot bind. + maxConcurrentRuns: agent.adapterType === "paperclip_runner" && + ["openai_dot", "grok_bot", "muse"].includes(String(parseObject(agent.adapterConfig).provider)) + ? 1 : normalizeMaxConcurrentRuns(heartbeat.maxConcurrentRuns), + skipTimerWhenNoActionableWork: asBoolean( + heartbeat.skipTimerWhenNoActionableWork ?? + heartbeat.requireActionableTimerWork ?? + heartbeat.issueOnlyTimer, + false, + ), + maxDailyRuns: normalizeOptionalNonNegativeInteger( + heartbeat.maxDailyRuns ?? + heartbeat.dailyRunLimit ?? + heartbeat.dailyRunCap ?? + heartbeat.maxRunsPerDay, + ), + maxDailyCostCents: normalizeOptionalNonNegativeInteger( + heartbeat.maxDailyCostCents ?? + heartbeat.dailyCostCentsLimit ?? + heartbeat.dailySpendCentsLimit ?? + heartbeat.dailyBudgetCents, + ), + }; + } + + function normalizeOptionalNonNegativeInteger(value: unknown) { + if (value === null || value === undefined || value === "") return null; + const normalized = Math.floor(asNumber(value, 0)); + return normalized >= 0 ? normalized : null; + } + + function currentUtcDayWindow(now = new Date()) { + const start = new Date( + Date.UTC( + now.getUTCFullYear(), + now.getUTCMonth(), + now.getUTCDate(), + 0, + 0, + 0, + 0, + ), + ); + const end = new Date( + Date.UTC( + now.getUTCFullYear(), + now.getUTCMonth(), + now.getUTCDate() + 1, + 0, + 0, + 0, + 0, + ), + ); + return { start, end }; + } + + async function getHeartbeatDailyCapBlock( + agent: typeof agents.$inferSelect, + policy: ReturnType, + options: { + checkRunCap?: boolean; + checkCostCap?: boolean; + excludeRunId?: string | null; + now?: Date; + } = {}, + client: Pick = db, + ) { + const checkRunCap = options.checkRunCap ?? true; + const checkCostCap = options.checkCostCap ?? true; + const { start, end } = currentUtcDayWindow(options.now); + if (checkRunCap && policy.maxDailyRuns !== null) { + const conditions = [ + eq(heartbeatRuns.companyId, agent.companyId), + eq(heartbeatRuns.agentId, agent.id), + gte(heartbeatRuns.startedAt, start), + lt(heartbeatRuns.startedAt, end), + notInArray(heartbeatRuns.status, ["queued", "scheduled_retry"]), + ]; + if (options.excludeRunId) { + conditions.push(sql`${heartbeatRuns.id} <> ${options.excludeRunId}`); + } + const [row] = await client + .select({ total: sql`count(*)::integer` }) + .from(heartbeatRuns) + .where(and(...conditions)); + const observed = Number(row?.total ?? 0); + if (observed >= policy.maxDailyRuns) { + return { + reason: "heartbeat.daily_run_limit", + observed, + limit: policy.maxDailyRuns, + }; + } + } + + if (checkCostCap && policy.maxDailyCostCents !== null) { + const [row] = await client + .select({ + total: sql`coalesce(sum(${costEvents.costCents}), 0)::text`, + }) + .from(costEvents) + .where( + and( + eq(costEvents.companyId, agent.companyId), + eq(costEvents.agentId, agent.id), + gte(costEvents.occurredAt, start), + lt(costEvents.occurredAt, end), + ), + ); + const observed = Number(row?.total ?? 0); + if (compareCents(String(row?.total ?? 0), policy.maxDailyCostCents) >= 0) { + return { + reason: "heartbeat.daily_cost_limit", + observed, + limit: policy.maxDailyCostCents, + }; + } + } + + return null; + } + + async function cancelQueuedRunForHeartbeatDailyCap( + run: typeof heartbeatRuns.$inferSelect, + dailyCapBlock: NonNullable< + Awaited> + >, + ) { + const now = new Date(); + const reason = + "Cancelled because the agent reached a per-day heartbeat budget cap before adapter invocation"; + const cancelled = await setRunStatus(run.id, "cancelled", { + finishedAt: now, + error: reason, + errorCode: dailyCapBlock.reason, + resultJson: { + ...parseObject(run.resultJson), + stopReason: dailyCapBlock.reason, + observed: dailyCapBlock.observed, + limit: dailyCapBlock.limit, + effectiveTimeoutSec: 0, + timeoutConfigured: false, + timeoutSource: "heartbeat_daily_cap_gate", + timeoutFired: false, + }, + }); + if (!cancelled) return null; + + await setWakeupStatus(run.wakeupRequestId, "skipped", { + finishedAt: now, + error: reason, + }); + + await appendRunEvent(cancelled, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: reason, + payload: { + reason: dailyCapBlock.reason, + observed: dailyCapBlock.observed, + limit: dailyCapBlock.limit, + }, + }); + + await releaseIssueExecutionAndPromote(cancelled, { + suppressImmediateRecovery: true, + }); + + return cancelled; + } + + async function hasActionableTimerWork(agent: typeof agents.$inferSelect) { + const row = await db + .select({ id: issues.id }) + .from(issues) + .where( + and( + eq(issues.companyId, agent.companyId), + eq(issues.assigneeAgentId, agent.id), + isNull(issues.assigneeUserId), + isNull(issues.hiddenAt), + inArray(issues.status, [...TIMER_ACTIONABLE_ISSUE_STATUSES]), + isNull(issues.conversationAgentId), + nonIdleSlackIssueCondition(), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + return Boolean(row); + } + + async function nextTimerHeartbeatAt(agent: typeof agents.$inferSelect, now = new Date()): Promise { + const policy = parseHeartbeatPolicy(agent); + if (!policy.enabled || policy.intervalSec <= 0) return null; + + if (policy.maxDailyRuns === 0 || policy.maxDailyCostCents === 0) return null; + if (policy.skipTimerWhenNoActionableWork && !(await hasActionableTimerWork(agent))) return null; + const dueAt = new Date(agent.lastHeartbeatAt ?? agent.createdAt).getTime() + policy.intervalSec * 1000; + const cap = await getHeartbeatDailyCapBlock(agent, policy, { now }); + if (!cap) return dueAt; + return Math.max(dueAt, currentUtcDayWindow(now).end.getTime()); + } + + async function markTimerHeartbeatChecked( + agentId: string, + source: WakeupOptions["source"], + ) { + if (source !== "timer") return; + await db.transaction(async tx => { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + const now = new Date(); + await tx.update(agents).set({ lastHeartbeatAt: now, updatedAt: now }) + .where(eq(agents.id, agentId)); + }); + } + + async function claimDueTimerHeartbeat( + agent: typeof agents.$inferSelect, + now: Date, + intervalSec: number, + ) { + const dueBefore = new Date(now.getTime() - intervalSec * 1000); + const claimed = await db.transaction(async tx => { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + return tx + .update(agents) + .set({ lastHeartbeatAt: now, updatedAt: now }) + .where(and( + eq(agents.id, agent.id), + eq(agents.companyId, agent.companyId), + or( + lte(agents.lastHeartbeatAt, dueBefore), + and(isNull(agents.lastHeartbeatAt), lte(agents.createdAt, dueBefore)), + ), + )) + .returning({ id: agents.id }) + .then((rows) => rows[0] ?? null); + }); + if (!claimed) return null; + return { wasFirstHeartbeat: !agent.lastHeartbeatAt }; + } + + function issueRunPriorityRank(priority: string | null | undefined) { + switch (priority) { + case "critical": + return 0; + case "high": + return 1; + case "medium": + return 2; + case "low": + return 3; + default: + return 4; + } + } + + async function listQueuedRunDependencyReadiness( + companyId: string, + queuedRuns: Array, + ) { + const issueIds = [ + ...new Set( + queuedRuns + .map((run) => + readNonEmptyString(parseObject(run.contextSnapshot).issueId), + ) + .filter((issueId): issueId is string => Boolean(issueId)), + ), + ]; + if (issueIds.length === 0) { + return new Map< + string, + Awaited> + >(); + } + return issuesSvc.listDependencyReadiness(companyId, issueIds); + } + + async function countRunningRunsForAgent(agentId: string) { + const [{ count }] = await db + .select({ count: sql`count(*)` }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.agentId, agentId), + eq(heartbeatRuns.status, "running"), + ), + ); + return Number(count ?? 0); + } + + async function withChatControlRecoveryGate( + run: typeof heartbeatRuns.$inferSelect, + stage: "claim" | "dispatch", + onClear: (tx: Db) => Promise, + nativeRecovery = false, + ) { + const issueId = readNonEmptyString( + parseObject(run.contextSnapshot).issueId, + ); + if (!issueId || run.invocationSource !== "automation") return onClear(db); + await options.beforeChatControlRecoveryCheck?.({ + runId: run.id, + issueId, + stage, + }); + if (stage === "dispatch" && nativeRecovery) { + // Protected historical/already-admitted owners did not previously take + // this fence's locks. A fresh exact read keeps unrelated Board contention + // from turning their recovery into a new admission failure. + const [current] = await db + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.agentId, run.agentId), + eq(heartbeatRuns.status, "running"), + ), + ) + .limit(1); + if ( + current && + current.wakeupRequestId === run.wakeupRequestId && + parseObject(current.contextSnapshot).issueId === issueId + ) { + const admission = readChatControlRecoveryAdmission(current); + const expected = readChatControlRecoveryAdmission(run); + if ( + expected !== "invalid" && + (admission === "admitted" || + (admission === "historical" && expected === "historical")) + ) + return onClear(db); + } + } + let terminal: typeof heartbeatRuns.$inferSelect | null = null; + try { + const attempt = () => db.transaction(async (tx) => { + terminal = null; + // Same queue-edit lock order, then the close committer's conversation + // row. NOWAIT releases partial locks on contention. Claim defers to the + // queue; dispatch retries this transaction before considering failure. + const [issue] = await tx + .select({ id: issues.id }) + .from(issues) + .where( + and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)), + ) + .for("update", { noWait: true }) + .limit(1); + if (!issue) return null; + if (run.wakeupRequestId) + await tx + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.id, run.wakeupRequestId), + eq(agentWakeupRequests.companyId, run.companyId), + eq(agentWakeupRequests.agentId, run.agentId), + ), + ) + .for("update", { noWait: true }) + .limit(1); + const [current] = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.agentId, run.agentId), + ), + ) + .for("update", { noWait: true }) + .limit(1); + if ( + !current || + current.status !== (stage === "claim" ? "queued" : "running") + ) + return null; + const admission = readChatControlRecoveryAdmission(current); + const expectedAdmission = readChatControlRecoveryAdmission(run); + const admissionLost = + expectedAdmission !== "historical" && admission === "historical"; + if ( + stage === "dispatch" && + nativeRecovery && + !admissionLost && + (admission === "historical" || admission === "admitted") + ) + return onClear(tx as unknown as Db); + // A new warm child can inherit a prior runner's PID during preparation. + // Those fields do not prove this run's turn was dispatched. + // Exact admitted/historical recovery retains its existing ownership; + // a marked, unadmitted bootstrap must earn admission even after crash. + const proof = + admission === "invalid" || admissionLost + ? { kind: "unresolved" as const } + : await readChatControlRecoveryStop( + tx as unknown as Db, + { + companyId: run.companyId, + issueId, + agentId: run.agentId, + sourceRunId: run.id, + }, + true, + ); + if (proof.kind === "clear") { + if (stage === "claim" || admission === "required") + await tx + .update(heartbeatRuns) + .set({ + runnerProfileJson: { + ...parseObject(current.runnerProfileJson), + [CHAT_CONTROL_RECOVERY_ADMISSION_KEY]: + chatControlRecoveryAdmission( + current, + stage === "dispatch" + ? "admitted" + : admission === "admitted" + ? "admitted" + : "required", + ), + }, + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, current.id)); + return onClear(tx as unknown as Db); + } + if (proof.kind === "unresolved" && stage === "claim") return null; + const now = new Date(); + const stopped = proof.kind === "stopped"; + const code = stopped + ? CHAT_CONTROL_RECOVERY_STOP_CODE + : CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE; + const error = stopped + ? "Automatic continuation stopped by the committed chat conversation close. Send a new request in chat or on the Board to start fresh work." + : "Automatic continuation source could not be verified before provider admission. Review the task and send a fresh request; this attempt will not automatically retry."; + await notifyChatRunMilestones(tx, { companyId: current.companyId, issueId }); + await notifyInstructionCollection(tx, { companyId: current.companyId, runId: current.id }); + await notifyExternalAgentSettlement(tx, { companyId: current.companyId, runId: current.id }); + await notifyCancelledWorkspacePreservation(tx, { companyId: current.companyId, runId: current.id }); + await notifyUnsafeWorkspaceExportRunChange(tx, current); + [terminal] = await tx + .update(heartbeatRuns) + .set({ + status: stopped ? "cancelled" : "failed", + errorCode: code, + error, + finishedAt: now, + resultJson: { + ...parseObject(current.resultJson), + automaticRecovery: { + code, + providerDispatched: false, + ...(stopped + ? { + sourceRunId: proof.sourceRunId, + conversationId: proof.conversationId, + publicationId: proof.publicationId, + } + : {}), + }, + }, + updatedAt: now, + }) + .where( + and( + eq(heartbeatRuns.id, current.id), + eq(heartbeatRuns.status, current.status), + ), + ) + .returning(); + if (!terminal) return null; + if (current.wakeupRequestId) + await tx + .update(agentWakeupRequests) + .set({ + status: stopped ? "skipped" : "failed", + error, + finishedAt: now, + updatedAt: now, + }) + .where( + and( + eq(agentWakeupRequests.id, current.wakeupRequestId), + eq(agentWakeupRequests.companyId, current.companyId), + eq(agentWakeupRequests.agentId, current.agentId), + eq(agentWakeupRequests.runId, current.id), + ne(agentWakeupRequests.status, "cancelled"), + ), + ); + await tx + .update(issues) + .set({ + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: now, + }) + .where( + and( + eq(issues.companyId, current.companyId), + eq(issues.id, issueId), + eq(issues.executionRunId, current.id), + ), + ); + return null; + }); + const result = stage === "dispatch" + ? await retryChatControlAdmission(attempt) + : await attempt(); + if (terminal) { + const settled = terminal as typeof heartbeatRuns.$inferSelect; + publishLiveEvent({ + companyId: settled.companyId, + type: "heartbeat.run.status", + payload: { + runId: settled.id, + agentId: settled.agentId, + status: settled.status, + errorCode: settled.errorCode, + error: settled.error, + }, + }); + publishRunLifecyclePluginEvent(settled); + if (stage === "dispatch") + await finalizeAgentStatus(settled.agentId, "cancelled"); + } + return result; + } catch (error) { + if (!isExternalChatWaitAuthorizationContention(error)) throw error; + if (stage === "dispatch") { + // No effect was admitted. Let the existing setup-failure path settle + // this attempt distinctly; it must not become a successful close. + throw new ChatControlRecoveryUnresolvedError(); + } + return null; + } + } + + async function claimQueuedRun( + run: typeof heartbeatRuns.$inferSelect, + companyAgents?: AgentOrgRow[], + deferredPostCommitEffects?: WakeQueuePostCommitEffect[], + ) { + if (run.status !== "queued") return run; + const [caseHandoffReceipt] = run.wakeupRequestId ? await db.select().from(caseHandoffs).where(and( + eq(caseHandoffs.wakeupRequestId, run.wakeupRequestId), eq(caseHandoffs.companyId, run.companyId), + )).limit(1) : []; + if (caseHandoffReceipt) { + const [current] = await db.select({ version: cases.handoffVersion, issueId: cases.workIssueId, + agentId: issues.assigneeAgentId, statusVersion: issues.statusVersion }).from(cases) + .innerJoin(issues, eq(issues.id, cases.workIssueId)).where(and(eq(cases.id, caseHandoffReceipt.caseId), + eq(cases.companyId, run.companyId), eq(issues.companyId, run.companyId))).limit(1); + if (!["pending_dispatch", "delivered"].includes(caseHandoffReceipt.state) || caseHandoffReceipt.toAgentId !== run.agentId || + !current || current.version !== caseHandoffReceipt.version || current.issueId !== caseHandoffReceipt.issueId || + current.agentId !== run.agentId || (!run.startedAt && current.statusVersion !== caseHandoffReceipt.appliedIssueStatusVersion)) { + const cancelled = await db.transaction(async tx => { + const [currentRun] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.status, "queued"))) + .for("update"); + if (!currentRun) return null; + await notifyChatRunMilestones(tx, { companyId: run.companyId, runId: run.id, + issueId: run.nativeIssueId ?? run.issueId ?? readNonEmptyString(parseObject(run.contextSnapshot).issueId) ?? undefined }); + await notifyInstructionCollection(tx, { companyId: run.companyId, runId: run.id }); + await notifyExternalAgentSettlement(tx, { companyId: run.companyId, runId: run.id }); + await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); + await notifyUnsafeWorkspaceExportRunChange(tx, run); + const [row] = await tx.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(), + errorCode: "case_handoff_superseded", error: "This case handoff was cancelled or superseded.", updatedAt: new Date() }) + .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.status, "queued"))).returning(); + return row; + }); + if (cancelled) await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: new Date(), error: cancelled.error }); + return null; + } + if (!(await instanceSettings.getExperimental()).enableCases) return null; + } + const caseIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); + if (caseIssueId) { + const [pending] = await db.select().from(caseHandoffs).where(and(eq(caseHandoffs.companyId, run.companyId), eq(caseHandoffs.issueId, caseIssueId), + inArray(caseHandoffs.state, ["awaiting_release", "pending_dispatch"]))).limit(1); + if (pending && (pending.wakeupRequestId !== run.wakeupRequestId || pending.state === "awaiting_release")) return null; + } + // Claim is the first execution boundary. startedAt survives same-run + // retries; restart recovery may also enter executeRun already running. + // Neither may reinterpret an admitted, already-started message as new work. + if (!run.startedAt && !(await slackMentionAllowsRunStart(db, run))) { + const now = new Date(); + const reason = "Message did not @mention the bot"; + // This runs under the agent start lock. Do not use cancelRunInternal, + // which recursively starts the next run and reacquires that lock. + const cancelled = await setRunStatus(run.id, "cancelled", { + finishedAt: now, + error: reason, + errorCode: "chat_mention_required", + }); + if (cancelled) { + await setWakeupStatus(run.wakeupRequestId, "skipped", { finishedAt: now, error: reason }); + await appendRunEvent(cancelled, { + eventType: "lifecycle", stream: "system", level: "warn", message: reason, + }); + await releaseIssueExecutionAndPromote(cancelled, { + suppressImmediateRecovery: true, deferredPostCommitEffects, + }); + } + return null; + } + + const agent = await getAgent(run.agentId); + if (!agent) { + await cancelRunInternal( + run.id, + "Cancelled because the agent no longer exists", + ); + return null; + } + // Keep accepted work in the durable queue until setup completes. + if (isAgentAwaitingSetup(agent)) return null; + const invokability = companyAgents + ? evaluateAgentInvokability(toAgentOrgRow(agent), companyAgents) + : await getAgentInvokability(agent); + if (!invokability.invokable) { + if (caseHandoffReceipt) return null; + await cancelRunInternal( + run.id, + `Cancelled because the agent is not invokable: ${invokability.reason}`, + ); + return null; + } + if (!run.startedAt && !await canStartNewExternalBotWork(agent)) return null; + + const context = parseObject(run.contextSnapshot); + const budgetBlock = await budgets.getInvocationBlock( + run.companyId, + run.agentId, + { + issueId: readNonEmptyString(context.issueId), + projectId: readNonEmptyString(context.projectId), + }, + ); + if (budgetBlock) { + if (caseHandoffReceipt) return null; + await cancelRunInternal(run.id, budgetBlock.reason); + return null; + } + + const dailyCapBlock = await getHeartbeatDailyCapBlock( + agent, + parseHeartbeatPolicy(agent), + { + excludeRunId: run.id, + checkRunCap: true, + checkCostCap: true, + }, + ); + if (dailyCapBlock) { + if (caseHandoffReceipt) return null; + await cancelQueuedRunForHeartbeatDailyCap(run, dailyCapBlock); + return null; + } + + const issueId = readNonEmptyString(context.issueId); + if (issueId && activeRunExecutions.size > 0) { + // Native finalization publishes success before workspace synchronization, + // provider suspension, and lease release finish. A queued comment must + // not acquire a fresh sandbox while its predecessor still owns that work. + // The executor's finally block retries this agent after removing its owner. + const [settlingOwner] = await db.select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where(and( + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.agentId, run.agentId), + eq(heartbeatRuns.runtimeMode, "native"), + inArray(heartbeatRuns.id, [...activeRunExecutions]), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, + )) + .limit(1); + if (settlingOwner) return null; + } + if (issueId) { + const activePauseHold = await treeControlSvc.getActivePauseHoldGate( + run.companyId, + issueId, + ); + const treeHoldInteractionWake = + activePauseHold && + (await isVerifiedIssueTreeControlInteractionWake(db, { + companyId: run.companyId, + issueId, + agentId: run.agentId, + runId: run.id, + wakeupRequestId: run.wakeupRequestId, + contextSnapshot: context, + })); + if (activePauseHold && !treeHoldInteractionWake) { + await cancelRunInternal( + run.id, + "Cancelled because issue is held by an active subtree pause hold", + ); + await logActivity(db, { + companyId: run.companyId, + actorType: "system", + actorId: "system", + agentId: run.agentId, + runId: run.id, + action: "issue.tree_hold_run_interrupted", + entityType: "heartbeat_run", + entityId: run.id, + issueId: issueId, + details: { + issueId, + holdId: activePauseHold.holdId, + rootIssueId: activePauseHold.rootIssueId, + source: "heartbeat.claim_queued_run", + securityPrinciples: [ + "Complete Mediation", + "Fail Securely", + "Secure Defaults", + ], + }, + }); + return null; + } + + const dependencyReadiness = await issuesSvc.listDependencyReadiness( + run.companyId, + [issueId], + ); + const readiness = dependencyReadiness.get(issueId); + const unresolvedBlockerCount = readiness?.unresolvedBlockerCount ?? 0; + if ( + unresolvedBlockerCount > 0 && + !allowsIssueInteractionWake( + context, + ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS, + ) + ) { + await cancelQueuedRunForBlockedDependencies( + run, + issueId, + readiness?.unresolvedBlockerIssueIds ?? [], + ); + logger.info( + { runId: run.id, issueId, unresolvedBlockerCount }, + "claimQueuedRun: cancelled blocked queued run", + ); + return null; + } + + const staleness = await runDispatch.cancelStaleQueuedRun({ + runId: run.id, + companyId: run.companyId, + expectedStatus: "queued", + }); + if (staleness.outcome === "cancelled" || staleness.outcome === "deferred") { + applyRunDispatchPostCommitEffects(staleness.postCommitEffects); + logger.info( + { + runId: run.id, + issueId, + outcome: staleness.outcome, + errorCode: staleness.outcome === "cancelled" ? staleness.errorCode : undefined, + }, + "claimQueuedRun: withheld queued run at the execution gate", + ); + return null; + } + } + + const claimedAt = new Date(); + const responsibleUserId = await resolveResponsibleUserIdForRun({ + run, + contextSnapshot: context, + issueContext: issueId + ? await getIssueExecutionContext(run.companyId, issueId) + : null, + routineEnvContext: { + routineId: null, + env: null, + responsibleUserId: null, + }, + }); + // All ordinary and comment claims use the same company-scoped issue + // lock. A batch may claim several runs before executeRun tracks any owner. + async function lockIssueExecutionClaim(tx: Db) { + const [owner] = issueId ? await tx.select({ + assigneeAgentId: issues.assigneeAgentId, + statusVersion: issues.statusVersion, + executionRunId: issues.executionRunId, + checkoutRunId: issues.checkoutRunId, + }).from(issues).where(and( + eq(issues.id, issueId), eq(issues.companyId, run.companyId), + )).for("update") : []; + const ownsIssue = owner?.assigneeAgentId === run.agentId && + context.wakeReason !== "source_scoped_recovery_action"; + const [handoff] = issueId ? await tx.select().from(caseHandoffs).where(and(eq(caseHandoffs.companyId, run.companyId), eq(caseHandoffs.issueId, issueId), + inArray(caseHandoffs.state, ["awaiting_release", "pending_dispatch"]))).limit(1) : []; + if (handoff && (handoff.wakeupRequestId !== run.wakeupRequestId || handoff.state === "awaiting_release")) + return { ownsIssue, blocked: true }; + if (caseHandoffReceipt) { + const [current] = await tx.select({ state: caseHandoffs.state, version: caseHandoffs.version, + currentVersion: cases.handoffVersion, workIssueId: cases.workIssueId, + appliedIssueStatusVersion: caseHandoffs.appliedIssueStatusVersion, + toAgentId: caseHandoffs.toAgentId, wakeupRequestId: caseHandoffs.wakeupRequestId }).from(caseHandoffs) + .innerJoin(cases, eq(cases.id, caseHandoffs.caseId)).where(and(eq(caseHandoffs.id, caseHandoffReceipt.id), + eq(caseHandoffs.companyId, run.companyId))).limit(1); + if (!current || !["pending_dispatch", "delivered"].includes(current.state) || current.version !== current.currentVersion || + current.workIssueId !== issueId || current.toAgentId !== run.agentId || current.wakeupRequestId !== run.wakeupRequestId || + !ownsIssue || (!run.startedAt && current.appliedIssueStatusVersion !== owner?.statusVersion)) + return { ownsIssue, blocked: true }; + } + if (ownsIssue && ["native_safe_replacement", "native_provider_overloaded"].includes(run.scheduledRetryReason ?? "") && + owner.checkoutRunId && owner.checkoutRunId !== run.id) { + return { ownsIssue, blocked: true }; + } + if (run.scheduledRetryReason === "native_provider_overloaded" && owner?.executionRunId && + owner.executionRunId !== run.id && owner.executionRunId !== run.retryOfRunId) { + return { ownsIssue, blocked: true }; + } + const previousRunId = run.scheduledRetryReason === "native_provider_overloaded" + ? run.retryOfRunId : ownsIssue ? owner?.executionRunId : null; + if (previousRunId && previousRunId !== run.id) { + const [previous] = await tx.select({ status: heartbeatRuns.status }) + .from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, previousRunId), + eq(heartbeatRuns.companyId, run.companyId), + )); + // A terminal result can precede workspace/lease cleanup on this or + // another controller. Local absence alone is not a release receipt. + if (!isHeartbeatRunTerminalStatus(previous?.status) || + liveRunExecutions.has(previousRunId)) { + return { ownsIssue, blocked: true }; + } + if (await hasPendingRunCleanup(tx, run.companyId, previousRunId)) { + return { ownsIssue, blocked: true }; + } + } + return { ownsIssue, blocked: false }; + } + async function bindClaimedIssueExecution(tx: Db, ownsIssue: boolean, claimedRun: typeof heartbeatRuns.$inferSelect | null | undefined) { + if (!claimedRun || !issueId || !ownsIssue) return; + await notifyStatusCardIssueChange(tx, run.companyId); + await tx.update(issues).set({ + executionRunId: claimedRun.id, + executionAgentNameKey: normalizeAgentNameKey(agent.name), + executionLockedAt: claimedAt, + updatedAt: claimedAt, + }).where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))); + } + const nativeReviewContext = readNativeReviewAssignmentContext(context); + const queuedCommentIds = queuedCommentIdsFromRunContext(context); + if ( + issueId && + run.invocationSource === "automation" && + queuedCommentIds.length > 0 + ) + await options.beforeChatControlRecoveryCheck?.({ + runId: run.id, + issueId, + stage: "claim", + }); + const queuedCommentClaim = + !nativeReviewContext && issueId && run.wakeupRequestId && queuedCommentIds.length > 0 + ? await db + .transaction(async (tx) => { + // Match the queue-edit lock order: issue, wake, then run. Once the + // run becomes running, a concurrent discard must observe the + // claimed wake and return an explicit conflict; if discard wins, + // this claim observes the cancelled queue and does no work. + const issueClaim = await lockIssueExecutionClaim(tx as unknown as Db); + if (issueClaim.blocked) return { kind: "stale" as const, run: null }; + const wake = await tx + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.id, run.wakeupRequestId!), + eq(agentWakeupRequests.companyId, run.companyId), + eq(agentWakeupRequests.agentId, run.agentId), + ), + ) + .for("update") + .limit(1) + .then((rows) => rows[0] ?? null); + const lockedRun = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.agentId, run.agentId), + ), + ) + .for("update") + .limit(1) + .then((rows) => rows[0] ?? null); + if ( + !wake || + wake.status !== "queued" || + wake.runId !== run.id || + !lockedRun || + lockedRun.status !== "queued" || + lockedRun.wakeupRequestId !== wake.id + ) { + return { kind: "stale" as const, run: null }; + } + + if (lockedRun.invocationSource === "automation") { + const admission = readChatControlRecoveryAdmission(lockedRun); + if (admission === "invalid") + return { kind: "stale" as const, run: null }; + const proof = await readChatControlRecoveryStop( + tx as unknown as Db, + { + companyId: lockedRun.companyId, + issueId, + agentId: lockedRun.agentId, + sourceRunId: lockedRun.id, + }, + true, + ); + if (proof.kind === "unresolved") + return { kind: "stale" as const, run: null }; + if (proof.kind === "stopped") { + const error = + "Automatic continuation stopped by the committed chat conversation close. Send a fresh request in chat or on the Board."; + await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); + await notifyInstructionCollection(tx, { companyId: run.companyId, runId: run.id }); + await notifyExternalAgentSettlement(tx, { companyId: run.companyId, runId: run.id }); + await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); + await notifyUnsafeWorkspaceExportRunChange(tx, run); + const [cancelled] = await tx + .update(heartbeatRuns) + .set({ + status: "cancelled", + errorCode: CHAT_CONTROL_RECOVERY_STOP_CODE, + error, + finishedAt: claimedAt, + resultJson: { + ...parseObject(lockedRun.resultJson), + automaticRecovery: { + code: CHAT_CONTROL_RECOVERY_STOP_CODE, + providerDispatched: false, + sourceRunId: proof.sourceRunId, + conversationId: proof.conversationId, + publicationId: proof.publicationId, + }, + }, + updatedAt: claimedAt, + }) + .where( + and( + eq(heartbeatRuns.id, lockedRun.id), + eq(heartbeatRuns.status, "queued"), + ), + ) + .returning(); + if (!cancelled) return { kind: "stale" as const, run: null }; + await tx + .update(agentWakeupRequests) + .set({ + status: "skipped", + error, + finishedAt: claimedAt, + updatedAt: claimedAt, + }) + .where(eq(agentWakeupRequests.id, wake.id)); + await tx + .update(issues) + .set({ + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: claimedAt, + }) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, run.companyId), + eq(issues.executionRunId, run.id), + ), + ); + return { kind: "cancelled" as const, run: cancelled }; + } + await tx + .update(heartbeatRuns) + .set({ + runnerProfileJson: { + ...parseObject(lockedRun.runnerProfileJson), + [CHAT_CONTROL_RECOVERY_ADMISSION_KEY]: + chatControlRecoveryAdmission( + lockedRun, + admission === "admitted" ? "admitted" : "required", + ), + }, + updatedAt: claimedAt, + }) + .where(eq(heartbeatRuns.id, lockedRun.id)); + } + const authoritativeIds = queuedCommentIdsFromWakePayload( + wake.payload, + ); + if (authoritativeIds.length === 0) { + // Legacy/direct comment wakes carry comment ids in their ordinary + // payload and context, not in the authoritative queued-message + // envelope. Preserve their established claim path; only an + // explicitly bound queued-message envelope is subject to the + // live-comment discard gate below. + await withExternalAdmissionGuard(tx, run.companyId, run.agentId, () => assertNoExternalOverlap(tx, run.companyId, run.agentId)); + await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); + const [claimedRun] = await tx + .update(heartbeatRuns) + .set({ + status: "running", + runnerProfileJson: sql`(case when jsonb_typeof(${heartbeatRuns.runnerProfileJson}) = 'object' then ${heartbeatRuns.runnerProfileJson} else '{}'::jsonb end) || ${JSON.stringify({ adapterDispatch: { adapterType: agent.adapterType } })}::jsonb`, + ...legacyControllerClaim(run.runtimeMode), + responsibleUserId, + startedAt: lockedRun.startedAt ?? claimedAt, + updatedAt: claimedAt, + }) + .where( + and( + eq(heartbeatRuns.id, lockedRun.id), + eq(heartbeatRuns.status, "queued"), + ), + ) + .returning(); + await bindClaimedIssueExecution(tx as unknown as Db, issueClaim.ownsIssue, claimedRun); + return claimedRun + ? { kind: "claimed" as const, run: claimedRun } + : { kind: "stale" as const, run: null }; + } + const commentRows = await tx + .select({ + id: issueComments.id, + deletedAt: issueComments.deletedAt, + }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, run.companyId), + eq(issueComments.issueId, issueId), + inArray(issueComments.id, authoritativeIds), + ), + ); + const liveIds = authoritativeIds.filter((commentId) => { + const comment = commentRows.find((row) => row.id === commentId); + return Boolean(comment && !comment.deletedAt); + }); + if (liveIds.length === 0) { + const reason = "Queued messages were discarded before dispatch"; + await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); + await notifyInstructionCollection(tx, { companyId: run.companyId, runId: run.id }); + await notifyExternalAgentSettlement(tx, { companyId: run.companyId, runId: run.id }); + await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); + await notifyUnsafeWorkspaceExportRunChange(tx, run); + const [cancelled] = await tx + .update(heartbeatRuns) + .set({ + status: "cancelled", + finishedAt: claimedAt, + error: reason, + errorCode: "queued_comment_discarded", + updatedAt: claimedAt, + }) + .where( + and( + eq(heartbeatRuns.id, lockedRun.id), + eq(heartbeatRuns.status, "queued"), + ), + ) + .returning(); + await tx + .update(agentWakeupRequests) + .set({ + status: "cancelled", + finishedAt: claimedAt, + error: reason, + updatedAt: claimedAt, + }) + .where(eq(agentWakeupRequests.id, wake.id)); + await tx + .update(issues) + .set({ + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: claimedAt, + }) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, run.companyId), + eq(issues.executionRunId, run.id), + ), + ); + return { + kind: "cancelled" as const, + run: cancelled ?? lockedRun, + }; + } + + await withExternalAdmissionGuard(tx, run.companyId, run.agentId, () => assertNoExternalOverlap(tx, run.companyId, run.agentId)); + await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); + await tx + .update(agentWakeupRequests) + .set({ + status: "claimed", + claimedAt, + payload: withQueuedCommentIdsInWakePayload( + wake.payload, + liveIds, + ), + updatedAt: claimedAt, + }) + .where(eq(agentWakeupRequests.id, wake.id)); + const [claimedRun] = await tx + .update(heartbeatRuns) + .set({ + status: "running", + runnerProfileJson: sql`(case when jsonb_typeof(${heartbeatRuns.runnerProfileJson}) = 'object' then ${heartbeatRuns.runnerProfileJson} else '{}'::jsonb end) || ${JSON.stringify({ adapterDispatch: { adapterType: agent.adapterType } })}::jsonb`, + ...legacyControllerClaim(run.runtimeMode), + responsibleUserId, + startedAt: lockedRun.startedAt ?? claimedAt, + contextSnapshot: withQueuedCommentIdsInRunContext( + lockedRun.contextSnapshot, + liveIds, + ), + updatedAt: claimedAt, + }) + .where( + and( + eq(heartbeatRuns.id, lockedRun.id), + eq(heartbeatRuns.status, "queued"), + ), + ) + .returning(); + await bindClaimedIssueExecution(tx as unknown as Db, issueClaim.ownsIssue, claimedRun); + return claimedRun + ? { kind: "claimed" as const, run: claimedRun } + : { kind: "stale" as const, run: null }; + }) + .catch((error) => { + if (isExternalChatWaitAuthorizationContention(error)) + return { kind: "stale" as const, run: null }; + throw error; + }) + : null; + if (queuedCommentClaim?.kind === "cancelled") { + await appendRunEvent(queuedCommentClaim.run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: + queuedCommentClaim.run.error ?? + "Queued messages were discarded before dispatch", + }); + publishLiveEvent({ + companyId: queuedCommentClaim.run.companyId, + type: "heartbeat.run.status", + payload: { + runId: queuedCommentClaim.run.id, + agentId: queuedCommentClaim.run.agentId, + status: queuedCommentClaim.run.status, + invocationSource: queuedCommentClaim.run.invocationSource, + triggerDetail: queuedCommentClaim.run.triggerDetail, + error: queuedCommentClaim.run.error ?? null, + errorCode: queuedCommentClaim.run.errorCode ?? null, + startedAt: queuedCommentClaim.run.startedAt + ? new Date(queuedCommentClaim.run.startedAt).toISOString() + : null, + finishedAt: queuedCommentClaim.run.finishedAt + ? new Date(queuedCommentClaim.run.finishedAt).toISOString() + : null, + }, + }); + publishRunLifecyclePluginEvent(queuedCommentClaim.run); + // Fire-and-forget: nothing else in this path depends on the emission, + // so it must not delay the return. + void emitAgentTaskRun(db, queuedCommentClaim.run); + return null; + } + const claimed = queuedCommentClaim + ? queuedCommentClaim.run + : await withChatControlRecoveryGate(run, "claim", async (tx) => { + const claimValues = { + status: "running", + runnerProfileJson: sql`(case when jsonb_typeof(${heartbeatRuns.runnerProfileJson}) = 'object' then ${heartbeatRuns.runnerProfileJson} else '{}'::jsonb end) || ${JSON.stringify({ adapterDispatch: { adapterType: agent.adapterType } })}::jsonb`, + ...legacyControllerClaim(run.runtimeMode), + responsibleUserId, + startedAt: run.startedAt ?? claimedAt, + updatedAt: claimedAt, + }; + if (nativeReviewContext) { + if (run.scheduledRetryReason === "native_provider_overloaded") { + const predecessor = await lockIssueExecutionClaim(tx); + if (predecessor.blocked) return null; + } + return claimQueuedNativeReviewRun(tx, { + run, claimedAt, claimValues, + agentNameKey: normalizeAgentNameKey(agent.name), + }); + } + return tx.transaction(async (claimTx) => { + const issueClaim = await lockIssueExecutionClaim(claimTx as unknown as Db); + if (issueClaim.blocked) return null; + if (run.wakeupRequestId) await claimTx.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.id, run.wakeupRequestId), eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId), + )).for("update"); + const [lockedRun] = await claimTx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.agentId, run.agentId), + )).for("update"); + if (!lockedRun || lockedRun.status !== "queued") return null; + await withExternalAdmissionGuard(claimTx, run.companyId, run.agentId, () => assertNoExternalOverlap(claimTx, run.companyId, run.agentId)); + await notifyChatRunMilestones(claimTx, { companyId: run.companyId, runId: run.id, issueId: issueId ?? undefined }); + const claimedRun = await claimTx.update(heartbeatRuns).set(claimValues).where(and( + eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "queued"), + )).returning().then((rows) => rows[0] ?? null); + await bindClaimedIssueExecution(claimTx as unknown as Db, issueClaim.ownsIssue, claimedRun); + return claimedRun; + }); + }); + if (!claimed) return null; + + publishLiveEvent({ + companyId: claimed.companyId, + type: "heartbeat.run.status", + payload: { + runId: claimed.id, + agentId: claimed.agentId, + status: claimed.status, + invocationSource: claimed.invocationSource, + triggerDetail: claimed.triggerDetail, + error: claimed.error ?? null, + errorCode: claimed.errorCode ?? null, + startedAt: claimed.startedAt + ? new Date(claimed.startedAt).toISOString() + : null, + finishedAt: claimed.finishedAt + ? new Date(claimed.finishedAt).toISOString() + : null, + }, + }); + publishRunLifecyclePluginEvent(claimed); + + if (!nativeReviewContext) { + await setWakeupStatus(claimed.wakeupRequestId, "claimed", { claimedAt }); + } + + // Fix A (lazy locking): stamp executionRunId now that the run is actually running, + // not at queue time. Guard is idempotent — safe if called more than once. + const claimedContext = parseObject(claimed.contextSnapshot); + const claimedIssueId = readNonEmptyString(claimedContext.issueId); + const claimedWakeReason = readNonEmptyString(claimedContext.wakeReason); + if ( + !nativeReviewContext && claimedIssueId && + claimedWakeReason !== "source_scoped_recovery_action" + ) { + const claimedAgent = await getAgent(claimed.agentId); + await db + .update(issues) + .set({ + executionRunId: claimed.id, + executionAgentNameKey: normalizeAgentNameKey(claimedAgent?.name), + executionLockedAt: claimedAt, + updatedAt: claimedAt, + }) + .where( + and( + eq(issues.id, claimedIssueId), + eq(issues.companyId, claimed.companyId), + // Mention/context runs can touch an issue, but only the current assignee + // owns the issue execution lock shown as the active run. + eq(issues.assigneeAgentId, claimed.agentId), + ["native_safe_replacement", "native_provider_overloaded"].includes(claimed.scheduledRetryReason ?? "") + ? or( + isNull(issues.checkoutRunId), + eq(issues.checkoutRunId, claimed.id), + ) + : undefined, + or( + isNull(issues.executionRunId), + eq(issues.executionRunId, claimed.id), + ), + ), + ); + } + + return claimed; + } + + // startNextQueuedRunForAgent checks admission suppression once, then claims + // runs (sets status "running"), then dispatches each to executeRun, which + // checks suppression again before it does any work. Suppression (task + // drain, worktree mode, a database restore) can start in the gap between + // those two checks. When executeRun's check catches that, the run is + // already claimed — release it back to "queued" so it does not keep a + // running execution lock that nothing will ever process. This runs inside + // the same promise startNextQueuedRunForAgent already tracks in + // activeRunExecutionPromises, so getTaskDrainStatus() keeps reporting the + // run as active until the release finishes. + // + // The run row, the wakeup request, and the issue execution lock all guard + // the same claim, so one transaction commits all three writes together. A + // partial write (for example the run flips to "queued" but the wakeup or + // issue update then fails) would let the execution promise clear from + // activeRunExecutionPromises while the wakeup stayed "claimed" or the + // issue stayed locked to a queued run — task-drain status would then read + // quiescent while the database still held part of the old claim. + async function releaseRunClaimedJustBeforeSuppression(runId: string) { + const now = new Date(); + await db.transaction(async (tx) => { + const [candidate] = await tx.select({ companyId: heartbeatRuns.companyId, contextSnapshot: heartbeatRuns.contextSnapshot }) + .from(heartbeatRuns).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.status, "running"))).for("update"); + if (!candidate) return; + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyChatRunMilestones(tx, { companyId: candidate.companyId, runId, + issueId: readNonEmptyString(parseObject(candidate.contextSnapshot).issueId) ?? undefined }); + const released = await tx + .update(heartbeatRuns) + .set({ + status: "queued", + startedAt: null, + responsibleUserId: null, + updatedAt: now, + }) + .where( + and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.status, "running")), + ) + .returning() + .then((rows) => rows[0] ?? null); + if (!released) return; + + if (released.wakeupRequestId) { + await tx + .update(agentWakeupRequests) + .set({ status: "queued", claimedAt: null, updatedAt: now }) + .where(and(eq(agentWakeupRequests.id, released.wakeupRequestId), ne(agentWakeupRequests.status, "cancelled"))); + } + + const context = parseObject(released.contextSnapshot); + const issueId = readNonEmptyString(context.issueId); + if (issueId) { + await tx + .update(issues) + .set({ + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: now, + }) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, released.companyId), + eq(issues.executionRunId, released.id), + ), + ); + } + }); + } + + async function cancelQueuedRunForBlockedDependencies( + run: typeof heartbeatRuns.$inferSelect, + issueId: string, + unresolvedBlockerIssueIds: string[], + ) { + const now = new Date(); + const reason = + "Cancelled because issue dependencies are still blocked; Paperclip will wake the assignee when blockers resolve"; + const cancelled = await setRunStatus(run.id, "cancelled", { + finishedAt: now, + error: reason, + errorCode: "issue_dependencies_blocked", + resultJson: { + ...parseObject(run.resultJson), + stopReason: "issue_dependencies_blocked", + effectiveTimeoutSec: 0, + timeoutConfigured: false, + timeoutSource: "dependency_gate", + timeoutFired: false, + }, + }); + if (!cancelled) return null; + + await setWakeupStatus(run.wakeupRequestId, "skipped", { + finishedAt: now, + error: reason, + }); + + await db.transaction(async tx => { + await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: run.companyId, issueId }); + await notifyNativeStatusReassessment(tx, { companyId: run.companyId, issueId }); + await tx + .update(issues) + .set({ + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: now, + }) + .where( + and( + eq(issues.companyId, run.companyId), + eq(issues.id, issueId), + eq(issues.executionRunId, run.id), + ), + ); + }); + + await appendRunEvent(cancelled, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: reason, + payload: { + issueId, + unresolvedBlockerIssueIds, + }, + }); + + return cancelled; + } + + async function resumeQueuedRuns() { + if ((await getSchedulingSuppression()).suppressed) return; + await resumeExecutionWaitComments(); + const cutoff = await getWorktreeExecutionCutoff(); + const pendingInterrupts = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId }) + .from(agentWakeupRequests).innerJoin(companies, eq(companies.id, agentWakeupRequests.companyId)) + .where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"), + eq(companies.status, "active"), + sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`, + lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)), + cutoff ? gte(agentWakeupRequests.requestedAt, cutoff) : undefined)) + .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); + for (const wake of pendingInterrupts) { + await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(and( + eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution"), + )); + await resumeQueuedCommentInterrupt(wake.companyId, wake.id).catch(err => { + logger.warn({ err, queueId: wake.id }, "failed to resume interrupted comment queue"); + }); + } + // A server restart or a message/cleanup race can leave a deferred wake + // after its owner has released the issue lock. Revisit it through the same + // release admission, so recovery holds and operator Stops still apply. + const strandedQueues = await db.select({ wake: agentWakeupRequests }) + .from(agentWakeupRequests) + .innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId), + sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`, + or(eq(issues.assigneeAgentId, agentWakeupRequests.agentId), + sql`${agentWakeupRequests.payload} #>> '{_paperclipWakeContext,wakeReason}' = 'issue_comment_mentioned'`))) + .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active"))) + .where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"), + isNull(issues.executionRunId), + or(and( + sql`jsonb_typeof(${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}') = 'array'`, + sql`${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`, + ), sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`), + sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is null`, + sql`NOT EXISTS (SELECT 1 FROM heartbeat_runs live_issue_run + WHERE live_issue_run.company_id = ${issues.companyId} + AND (live_issue_run.context_snapshot->>'issueId' = ${issues.id}::text + OR live_issue_run.native_issue_id = ${issues.id}) + AND live_issue_run.status IN ('queued', 'running', 'scheduled_retry'))`, + cutoff ? gte(agentWakeupRequests.requestedAt, cutoff) : undefined)) + .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); + const releasedNativeSources = new Set(); + for (const { wake } of strandedQueues) { + if (!queuedCommentIdsFromWakePayload(wake.payload).length && + !await readQueuedInteractionResponse(db, wake.companyId, String(wake.payload?.issueId), wake.payload)) continue; + const [latest] = await db.select({ + id: heartbeatRuns.id, + companyId: heartbeatRuns.companyId, + status: heartbeatRuns.status, + finishedAt: heartbeatRuns.finishedAt, + startedAt: heartbeatRuns.startedAt, + runtimeMode: heartbeatRuns.runtimeMode, + errorCode: heartbeatRuns.errorCode, + sourceVersion: sql`${heartbeatRuns}.xmin::text`, + }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, wake.companyId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${String(wake.payload?.issueId)}`, + or(isNull(heartbeatRuns.nativeIssueId), sql`${heartbeatRuns.nativeIssueId}::text = ${String(wake.payload?.issueId)}`), + )).orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); + await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(and( + eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution"), + )); + if (!latest || !latest.finishedAt || !isHeartbeatRunTerminalStatus(latest.status)) continue; + if (latest.runtimeMode !== "legacy" && latest.runtimeMode !== "native") continue; + if (wake.idempotencyKey?.startsWith("chat-inbound:")) { + await resumeDeferredChatWake(db, { companyId: wake.companyId, agentId: wake.agentId, + issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(() => { + logger.warn({ requestId: wake.id }, "saved chat source could not resume after recovery"); + }); + continue; + } + const releaseSource = `${latest.companyId}:${String(wake.payload?.issueId)}:${latest.id}:${latest.sourceVersion}`; + if (latest.runtimeMode === "native" && releasedNativeSources.has(releaseSource)) continue; + if (latest.runtimeMode === "legacy") { + const cancelledAdmission = latest.status === "cancelled" && !latest.startedAt && + latest.errorCode === "execution_reconciliation_required"; + if ((latest.status !== "cancelled" || cancelledAdmission) && await getExecutionBlocker(db, wake.companyId, String(wake.payload?.issueId))) { + await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => { + logger.warn({ err, queueId: wake.id }, "failed to deliver saved legacy comment after recovery stopped"); + }); + } + } + await releaseIssueExecutionAndPromote(latest, { suppressImmediateRecovery: true, resumeStrandedQueue: true }).then(() => { + if (latest.runtimeMode === "native") releasedNativeSources.add(releaseSource); + }).catch(err => { + logger.warn({ err, queueId: wake.id }, "failed to promote stranded comments"); + }); + } + + // The cancellation marker is durable intent. Retry while its exact queue + // is still deferred, including after a failed cleanup promotion or restart. + // Normal admission still checks process ownership, leases, pauses, and scope. + const interruptedQueues = await db + .select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId }) + .from(agentWakeupRequests) + .innerJoin(heartbeatRuns, and( + sql`${heartbeatRuns.resultJson}->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text`, + eq(heartbeatRuns.companyId, agentWakeupRequests.companyId), + eq(heartbeatRuns.agentId, agentWakeupRequests.agentId), + )) + .innerJoin(companies, eq(companies.id, heartbeatRuns.companyId)) + .where(and( + eq(agentWakeupRequests.status, "deferred_issue_execution"), + eq(heartbeatRuns.status, "cancelled"), + eq(heartbeatRuns.runtimeMode, "legacy"), + eq(companies.status, "active"), + cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined, + )); + for (const run of interruptedQueues) { + await releaseIssueExecutionAndPromote(run, { suppressImmediateRecovery: true }).catch((err) => { + logger.error({ err, runId: run.id }, "failed to retry interrupted comment queue"); + }); + } + + await resumeQueuedRunsAfterDrain(); + } + + // Release admission promptly without first scanning unrelated recovery work. + // Every run still passes the ordinary per-agent claim and dispatch gates. + async function resumeQueuedRunsAfterDrain(scope?: { companyId: string }) { + if ((await getSchedulingSuppression()).suppressed) return; + const cutoff = await getWorktreeExecutionCutoff(); + const queuedRuns = await db + .select({ agentId: heartbeatRuns.agentId }) + .from(heartbeatRuns) + .innerJoin(companies, eq(companies.id, heartbeatRuns.companyId)) + .where( + and( + eq(heartbeatRuns.status, "queued"), + scope ? eq(heartbeatRuns.companyId, scope.companyId) : undefined, + eq(companies.status, "active"), + cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined, + ), + ); + + const agentIds = [...new Set(queuedRuns.map((r) => r.agentId))]; + for (const agentId of agentIds) { + await startNextQueuedRunForAgent(agentId); + } + } + + // A 403 from claimQueuedRun comes from the run's own persisted identity + // (an unverifiable interrupt receipt, a manual wake with no user). Those rows + // do not change, so the claim fails the same way on every pass and restart. + function isPermanentClaimRejection(err: unknown): err is HttpError { + return err instanceof HttpError && err.status === 403; + } + + // Other 4xx rejections can clear later (a responsible user gets assigned, a + // conflicting claim finishes). Keep the run queued, but do not let it stop + // the rest of the queue or startup recovery. + function isDeferrableClaimRejection(err: unknown): err is HttpError { + return err instanceof HttpError && err.status >= 400 && err.status < 500; + } + + // Settle runs that can never be claimed. Letting the error escape stalls the + // agent's queue and, during startup recovery, stops the server from booting. + async function cancelRejectedQueuedRuns( + rejected: Array<{ run: typeof heartbeatRuns.$inferSelect; err: HttpError }>, + ) { + for (const { run, err } of rejected) { + logger.warn( + { err, runId: run.id, agentId: run.agentId, companyId: run.companyId }, + "cancelling queued heartbeat run whose claim was rejected", + ); + try { + await cancelRunInternal( + run.id, + `Cancelled because the queued run cannot be claimed: ${err.message}`, + { errorCode: "queued_run_claim_rejected" }, + ); + } catch (cancelErr) { + logger.error( + { err: cancelErr, runId: run.id }, + "failed to cancel queued heartbeat run whose claim was rejected; it stays queued for the next recovery pass", + ); + } + } + } + + async function startNextQueuedRunForAgent(agentId: string) { + if ((await getSchedulingSuppression()).suppressed) return []; + const cutoff = await getWorktreeExecutionCutoff(); + // Cancelled after the start lock is released: cancelRunInternal promotes the + // agent's next queued run, which takes this same lock. + const rejectedClaims: Array<{ run: typeof heartbeatRuns.$inferSelect; err: HttpError }> = []; + const deferredPostCommitEffects: WakeQueuePostCommitEffect[] = []; + let cancellationReason: string | undefined; + + return withAgentStartLock(agentId, async () => { + const agent = await getAgent(agentId); + if (!agent) return []; + const invokability = await getAgentInvokability(agent); + if (!invokability.invokable) { + if (shouldCancelRunsForNonInvokableAgent(invokability)) { + cancellationReason = `Cancelled because the agent is not invokable: ${invokability.reason}`; + } + return []; + } + const policy = parseHeartbeatPolicy(agent); + const runningCount = await countRunningRunsForAgent(agentId); + const availableSlots = Math.max( + 0, + policy.maxConcurrentRuns - runningCount, + ); + if (availableSlots <= 0) return []; + + const queuedRuns = await db + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.agentId, agentId), + eq(heartbeatRuns.status, "queued"), + cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined, + ), + ) + .orderBy(asc(heartbeatRuns.createdAt)); + if (queuedRuns.length === 0) return []; + if (queuedRuns.every(run => !run.startedAt) && !await canStartNewExternalBotWork(agent)) return []; + + const dependencyReadiness = await listQueuedRunDependencyReadiness( + agent.companyId, + queuedRuns, + ); + const queuedIssueIds = [ + ...new Set( + queuedRuns + .map((run) => + readNonEmptyString(parseObject(run.contextSnapshot).issueId), + ) + .filter((issueId): issueId is string => Boolean(issueId)), + ), + ]; + const issueRows = await db + .select({ + id: issues.id, + status: issues.status, + priority: issues.priority, + }) + .from(issues) + .where( + queuedIssueIds.length > 0 + ? and( + eq(issues.companyId, agent.companyId), + inArray(issues.id, queuedIssueIds), + ) + : sql`false`, + ); + const issueById = new Map(issueRows.map((row) => [row.id, row])); + const companyAgents = await listCompanyAgentOrgRows(agent.companyId); + const prioritizedRuns = [...queuedRuns].sort((left, right) => { + const leftIssueId = readNonEmptyString( + parseObject(left.contextSnapshot).issueId, + ); + const rightIssueId = readNonEmptyString( + parseObject(right.contextSnapshot).issueId, + ); + const leftReadiness = leftIssueId + ? dependencyReadiness.get(leftIssueId) + : null; + const rightReadiness = rightIssueId + ? dependencyReadiness.get(rightIssueId) + : null; + const leftReady = leftIssueId + ? (leftReadiness?.isDependencyReady ?? true) + : true; + const rightReady = rightIssueId + ? (rightReadiness?.isDependencyReady ?? true) + : true; + const leftIssue = leftIssueId ? issueById.get(leftIssueId) : null; + const rightIssue = rightIssueId ? issueById.get(rightIssueId) : null; + const leftRank = leftIssueId + ? leftReady + ? leftIssue?.status === "in_progress" + ? 0 + : 1 + : 3 + : 2; + const rightRank = rightIssueId + ? rightReady + ? rightIssue?.status === "in_progress" + ? 0 + : 1 + : 3 + : 2; + if (leftRank !== rightRank) return leftRank - rightRank; + const leftPriorityRank = issueRunPriorityRank(leftIssue?.priority); + const rightPriorityRank = issueRunPriorityRank(rightIssue?.priority); + if (leftPriorityRank !== rightPriorityRank) + return leftPriorityRank - rightPriorityRank; + return left.createdAt.getTime() - right.createdAt.getTime(); + }); + + const claimedRuns: Array = []; + for (const queuedRun of prioritizedRuns) { + if (claimedRuns.length >= availableSlots) break; + let claimed: typeof heartbeatRuns.$inferSelect | null; + try { + claimed = await claimQueuedRun(queuedRun, companyAgents, deferredPostCommitEffects); + } catch (err) { + if (isPermanentClaimRejection(err)) { + rejectedClaims.push({ run: queuedRun, err }); + continue; + } + if (!isDeferrableClaimRejection(err)) throw err; + logger.warn( + { err, runId: queuedRun.id, agentId: queuedRun.agentId, companyId: queuedRun.companyId }, + "queued heartbeat run claim was rejected; leaving it queued for the next recovery pass", + ); + continue; + } + if (claimed) claimedRuns.push(claimed); + } + if (claimedRuns.length === 0) return []; + + for (const claimedRun of claimedRuns) { + const execution = executeRun(claimedRun.id).catch((err) => { + logger.error( + { err, runId: claimedRun.id }, + "queued heartbeat execution failed", + ); + }); + // Register the in-flight execution so drainActiveRunExecutions() can await + // it. executeRun resolves only after its finally block finishes flushing + // run rows/events, so awaiting this promise guarantees the run's writes + // have landed before a caller (e.g. a test's afterEach) mutates the DB. + activeRunExecutionPromises.add(execution); + void execution.finally(() => { + // drainActiveRunExecutions loops on activeRunExecutionPromises.size, + // so an entry that never clears here would hang it forever. + activeRunExecutionPromises.delete(execution); + }); + } + return claimedRuns; + }).finally(async () => { + // Dispatch promoted inputs after the agent start lock is released. + try { + await applyWakeQueuePostCommitEffects(deferredPostCommitEffects); + } finally { + if (cancellationReason) await cancelActiveForAgentInternal(agentId, cancellationReason); + await cancelRejectedQueuedRuns(rejectedClaims); + } + }); + } + + // Public wakeup entry point. Callers dispatch it fire-and-forget, so register + // the promise in activeWakeupPromises before it starts its asynchronous + // prologue. drainActiveRunExecutions can then await a wake that is still before + // run registration. Internal callers reference enqueueWakeup directly and + // already await it, so they do not need this registration. + function trackWakeup( + agentId: string, + opts: WakeupOptions = {}, + ): ReturnType { + const promise = enqueueWakeup(agentId, opts); + activeWakeupPromises.add(promise); + void promise + .catch(() => {}) + .finally(() => { + activeWakeupPromises.delete(promise); + }); + return promise; + } + + async function enqueueWakeup(agentId: string, opts: WakeupOptions = {}, executionWaitRequestId?: string) { + const source = opts.source ?? "on_demand"; + const triggerDetail = opts.triggerDetail ?? null; + const contextSnapshot: Record = { + ...(opts.contextSnapshot ?? {}), + }; + const reason = opts.reason ?? null; + let payload = opts.payload ? { ...opts.payload } : null; + // Only the board queue route can record interruption authority on an + // existing receipt. Never accept this internal marker from a wake caller. + if (payload) { + delete payload.queuedCommentInterrupt; + delete payload.manualUserWake; + delete payload.taskRetry; + } + if (opts.manualUserWake) { + if (opts.requestedByActorType !== "user" || !opts.requestedByActorId || opts.failedRunId) { + throw new HttpError(403, "Manual wake requires an authenticated user"); + } + payload = { ...payload, manualUserWake: true }; + } + const executionReconciliationWake = + contextSnapshot.source === "execution.reconciled" || + opts.idempotencyKey?.startsWith("execution-reconciliation:") === true; + const { + contextSnapshot: enrichedContextSnapshot, + issueIdFromPayload, + taskKey, + wakeCommentId, + } = enrichWakeContextSnapshot({ + contextSnapshot, + reason, + source, + triggerDetail, + payload, + }); + // Keep each request's own server-derived origin, including coalesced wakes. + // A run's merged context cannot establish which caller authored one receipt. + // Overwrite caller-supplied nested context rather than trusting it. + payload = { ...payload, [DEFERRED_WAKE_CONTEXT_KEY]: { ...enrichedContextSnapshot } }; + let issueId = + readNonEmptyString(enrichedContextSnapshot.issueId) ?? issueIdFromPayload; + if (executionReconciliationWake && !issueId) return null; + const durableRequest = opts.durableChatRequest; + let taskRetryActorId: string | null = null; + delete enrichedContextSnapshot.boardMessageRetry; + if (opts.taskRetryRequestId) { + const [receipt] = await db.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.id, opts.taskRetryRequestId), eq(agentWakeupRequests.agentId, agentId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + eq(agentWakeupRequests.requestedByActorType, "user"), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId ?? ""}`, + )); + const retry = readTaskRetryIntent(receipt?.payload); + if (!receipt || retry?.generationId !== receipt.id || retry.canonicalIntentId || + retry.actorId !== receipt.requestedByActorId) return null; + const chatRetry = retry.chatActionId && durableRequest?.id === retry.chatActionId && + durableRequest.failedRunRetry?.failedRunId === retry.sourceRunId; + if (retry.chatActionId ? !chatRetry : + opts.requestedByActorType !== "user" || retry.actorId !== opts.requestedByActorId) return null; + taskRetryActorId = retry.actorId; + enrichedContextSnapshot.forceFreshSession = true; + enrichedContextSnapshot.previousRunId = retry.sourceRunId; + enrichedContextSnapshot.taskRetryRequestId = receipt.id; + if (retry.boardMessageOrigin) enrichedContextSnapshot.boardMessageRetry = true; + } + + + let agent = await getAgent(agentId); + if (!agent) throw notFound("Agent not found"); + if (issueId) { + const [pending] = await db.select().from(caseHandoffs).where(and(eq(caseHandoffs.companyId, agent.companyId), eq(caseHandoffs.issueId, issueId), + inArray(caseHandoffs.state, ["awaiting_release", "pending_dispatch"]))).limit(1); + if (pending && opts.durableCaseHandoffRequest?.handoffId !== pending.id) return null; + } + // Mentions only annotate comments. Ignore legacy callers before creating + // a run or deferred request; assignment and review have their own wakes. + if (reason === "issue_comment_mentioned" || enrichedContextSnapshot.wakeReason === "issue_comment_mentioned") return null; + if (issueId) { + const conversation = await getIssueExecutionContext(agent.companyId, issueId); + if (taskRetryActorId && conversation) await assertTaskRetryConversationAllowed(db, conversation, taskRetryActorId); + if (reason === "issue_children_completed" && conversation?.originKind === "onboarding_first_task") enrichedContextSnapshot.onboardingCompletion = true; + if (isConversation(conversation)) { + if (opts.manualUserWake && conversation!.conversationUserId !== opts.requestedByActorId) { + throw new HttpError(403, "Only the conversation owner can start a chat run"); + } + if (isConversationExecutionWake(conversation, reason ?? readNonEmptyString(enrichedContextSnapshot.wakeReason))) return null; + if (agent.id !== conversation!.conversationAgentId) return null; + if (!(await instanceSettings.getExperimental()).enableAgentChat) return null; + if (!wakeCommentId && !opts.taskRetryRequestId && isWaitingConversation(conversation) && !hasInteractionContinuationWakeContext(enrichedContextSnapshot) && reason !== CHAT_COMPLETION_WAKE_REASON) return null; + } + } + if (agent.adapterType === "paperclip_runner") { + const oldConfig = parseObject(agent.adapterConfig); + const nextConfig = normalizeLegacyRunnerProvider(oldConfig); + if (nextConfig !== oldConfig) { + await agentService(db).update( + agent.id, + { adapterConfig: nextConfig }, + { + recordRevision: { + source: "normalize_runner_provider", + createdByAgentId: null, + createdByUserId: null, + }, + }, + ); + await logActivity(db, { + companyId: agent.companyId, + actorType: "system", + actorId: "heartbeat", + action: "agent.updated", + entityType: "agent", + entityId: agent.id, + details: { provider: "codex", reason: "native_codex_provider" }, + }); + agent = (await getAgent(agentId))!; + } + } + + if (opts.failedRunId) { + const failed = await getRun(opts.failedRunId, { includeExecutionEvidence: true }); + if (opts.requestedByActorType !== "user" || !opts.requestedByActorId || + reason !== "retry_failed_run" || source !== "on_demand" || triggerDetail !== "manual" || + !failed || failed.companyId !== agent.companyId || failed.agentId !== agentId || + !(await canRetryStoppedRun(db, failed)) || + (failed.nativeIssueId ?? readNonEmptyString(failed.contextSnapshot?.issueId)) !== issueId) { + throw conflict("The selected failed run cannot be retried for this task."); + } + if (!activeRunExecutions.has(failed.id) && !adapterExecutionControls.has(failed.id)) { + await sweepPendingCleanupLeases({ explicitRetry: { + companyId: failed.companyId, runId: failed.id, actorId: opts.requestedByActorId, + } }); + } + if (isConversationAdapter(agent.adapterType) || agent.adapterType === "paperclip_runner") { + enrichedContextSnapshot.previousRunId = failed.id; + enrichedContextSnapshot.forceFreshSession = true; + } + } + + const caseRequest = opts.durableCaseHandoffRequest; + const waitForCaseAdmission = (reason: string, message: string): never => { + // The case ledger is the outbox. Roll back this admission attempt rather + // than consume its one receipt ID in a generic skipped/deferred queue. + throw conflict(message, { code: "case_handoff_admission_wait", reason }); + }; + let caseReceipt: { id: string; requestedAt: Date } | undefined; + if (caseRequest) { + if (opts.taskRetryRequestId || opts.durableChatRequest || opts.durableDotRequest || opts.durableMuseRequest || source !== "assignment" || reason !== "case_handoff") + throw conflict("Invalid case handoff wake authority"); + const [handoff] = await db.select().from(caseHandoffs).where(and(eq(caseHandoffs.id, caseRequest.handoffId), + eq(caseHandoffs.companyId, agent.companyId), eq(caseHandoffs.issueId, caseRequest.issueId), + eq(caseHandoffs.toAgentId, agentId), eq(caseHandoffs.wakeupRequestId, caseRequest.wakeupRequestId), + inArray(caseHandoffs.state, ["pending_dispatch", "delivered"]))).limit(1); + if (!handoff || issueId !== handoff.issueId || caseRequest.expectedIssueStatusVersion !== handoff.appliedIssueStatusVersion) + throw conflict("Case handoff no longer matches its task assignment"); + caseReceipt = { id: handoff.wakeupRequestId, requestedAt: handoff.createdAt }; + opts = { ...opts, allowRunCoalescing: false, idempotencyKey: `case-handoff:${handoff.id}`, + issueStateGuard: { assigneeAgentId: agentId, statuses: ["todo", "in_progress"], statusVersion: caseRequest.expectedIssueStatusVersion } }; + } + if (durableRequest) { + assertDurableChatWakeupRequest(durableRequest, { + agentId, + companyId: agent.companyId, + issueId, + commentId: wakeCommentId ?? null, + requestedByActorType: opts.requestedByActorType, + requestedByActorId: opts.requestedByActorId, + }); + opts = { ...opts, idempotencyKey: durableRequest.idempotencyKey }; + } + if ( + Object.hasOwn(enrichedContextSnapshot, "chatFailedRunRetry") && + !durableRequest?.failedRunRetry + ) { + throw new FailedChatRunRetryAuthorizationError(); + } + if (durableRequest?.failedRunRetry) { + opts = { ...opts, allowRunCoalescing: false }; + } + const dotRequest = opts.durableDotRequest; + const museRequest = opts.durableMuseRequest; + if (dotRequest && (durableRequest || museRequest || dotRequest.agentId !== agentId || + dotRequest.companyId !== agent.companyId || dotRequest.issueId !== issueId || + dotRequest.requestId !== payload?.dotRequestId || source !== "assignment" || + opts.requestedByActorType !== "agent" || opts.requestedByActorId !== agentId || + agent.adapterType !== "paperclip_runner" || !["openai_dot", "grok_bot"].includes(String(parseObject(agent.adapterConfig).provider)) || + opts.idempotencyKey !== dotRequest.idempotencyKey)) { + throw conflict("Dot work request does not match its admission authority."); + } + if (museRequest && (durableRequest || dotRequest || museRequest.agentId !== agentId || + museRequest.companyId !== agent.companyId || museRequest.issueId !== issueId || + museRequest.requestId !== payload?.museRequestId || source !== "assignment" || + opts.requestedByActorType !== "agent" || opts.requestedByActorId !== agentId || + agent.adapterType !== "paperclip_runner" || parseObject(agent.adapterConfig).provider !== "muse" || + parseObject(agent.adapterConfig).museBindingId !== museRequest.bindingId || + !Number.isSafeInteger(museRequest.bindingGeneration) || museRequest.bindingGeneration < 1 || + payload?.museBindingId !== museRequest.bindingId || payload?.museBindingGeneration !== museRequest.bindingGeneration || + payload?.museRequestDigest !== museRequest.requestDigest || !/^sha256:[a-f0-9]{64}$/.test(museRequest.requestDigest) || + opts.idempotencyKey !== museRequest.idempotencyKey)) { + throw conflict("Muse work request does not match its admission authority."); + } + const receiptRequest = durableRequest ?? dotRequest ?? museRequest ?? caseReceipt; + const durableReceiptFields = receiptRequest + ? { id: receiptRequest.id, requestedAt: receiptRequest.requestedAt } + : {}; + const existingDurableReceipt = async (queryDb: Db) => { + if (!receiptRequest) return null; + const receipt = await queryDb + .select() + .from(agentWakeupRequests) + .where(eq(agentWakeupRequests.id, receiptRequest.id)) + .limit(1) + .then((rows) => rows[0] ?? null); + if (receipt && durableRequest) assertDurableChatWakeupReceipt(durableRequest, receipt); + if (receipt && caseRequest && (receipt.companyId !== agent.companyId || receipt.agentId !== agentId || + receipt.payload?.issueId !== caseRequest.issueId || receipt.idempotencyKey !== `case-handoff:${caseRequest.handoffId}`)) + throw conflict("Case handoff wake receipt identity changed"); + if (receipt && dotRequest && (receipt.companyId !== dotRequest.companyId || + receipt.agentId !== dotRequest.agentId || receipt.source !== "assignment" || + receipt.requestedByActorType !== "agent" || receipt.requestedByActorId !== dotRequest.agentId || + receipt.idempotencyKey !== dotRequest.idempotencyKey || receipt.payload?.issueId !== dotRequest.issueId || + receipt.payload?.dotRequestId !== dotRequest.requestId)) { + throw conflict("requestId was reused for another task."); + } + if (receipt && museRequest && (receipt.companyId !== museRequest.companyId || + receipt.agentId !== museRequest.agentId || receipt.source !== "assignment" || + receipt.requestedByActorType !== "agent" || receipt.requestedByActorId !== museRequest.agentId || + receipt.idempotencyKey !== museRequest.idempotencyKey || receipt.payload?.issueId !== museRequest.issueId || + receipt.payload?.museRequestId !== museRequest.requestId || receipt.payload?.museBindingId !== museRequest.bindingId || + receipt.payload?.museBindingGeneration !== museRequest.bindingGeneration || receipt.payload?.museRequestDigest !== museRequest.requestDigest)) { + throw conflict("Request ID reused with changed input."); + } + if (receipt && durableRequest && receipt.status === "deferred_issue_execution" && !receipt.runId && + ((durableRequest.failedRunRetry && opts.taskRetryRequestId && + receipt.payload?.taskRetryRequestId === opts.taskRetryRequestId) || + (durableRequest.resumeDeferred && !durableRequest.failedRunRetry && + receipt.payload?.issueId === durableRequest.issueId && + queuedCommentIdsFromWakePayload(receipt.payload).includes(durableRequest.commentId)))) { + // The live inbound owner or a bound Retry may reconsider the same saved receipt. + // The issue lock, live chat authority and execution holds below still + // decide whether it can dispatch; this never creates another wake. + executionWaitRequestId = receipt.id; + return null; + } + return receipt; + }; + const priorReceipt = await existingDurableReceipt(db); + if (priorReceipt) { + // Replaying an admission receipt is not fresh authority to dispatch it. + // The normal queue owns dispatch and its current execution-policy checks. + return priorReceipt.runId ? getRun(priorReceipt.runId) : null; + } + + const agentDebug = parseObject(parseObject(agent.runtimeConfig).debug); + const runDebug = parseObject(enrichedContextSnapshot.debug); + if ( + agentDebug.providerTrace === "raw" && + runDebug.providerTrace !== "raw" + ) { + enrichedContextSnapshot.debug = { + ...runDebug, + providerTrace: "raw", + }; + enrichedContextSnapshot.providerTraceRequestedBy = `agent:${agent.id}:debug-setting`; + enrichedContextSnapshot.providerTraceRequestSource = + "agent_debug_setting"; + } + + // Automatic signals are replaceable; user input and interaction delivery + // retain distinct durable receipts even when the same gate blocks them. + const coalesceExecutionWait = + opts.requestedByActorType === "system" && + !receiptRequest && + !wakeCommentId && + queuedCommentIdsFromRunContext(enrichedContextSnapshot).length === 0 && + !isInteractionResolutionWakePayload(payload ?? {}) && + !hasInteractionContinuationWakeContext(enrichedContextSnapshot); + const writeSkippedRequest = async ( + skipReason: string, + patch: Partial = {}, + waitCondition?: Record, + ) => { + // A temporary refusal must leave the handoff retryable, without burning + // its one durable receipt ID on a skipped wake. + if (caseRequest) return { created: false }; + if (executionWaitRequestId) { + await db.update(agentWakeupRequests).set({ + payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', + coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify({ + reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active" + ? "This task is paused. Resume it to send your saved message." + : "Waiting for task execution to be enabled. Your message is saved."), + })}::jsonb)`, + updatedAt: new Date(), + }).where(and(eq(agentWakeupRequests.id, executionWaitRequestId), + eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId), + eq(agentWakeupRequests.status, "deferred_issue_execution"))); + return { created: false }; + } + const request = { + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: skipReason, + payload, + status: "skipped", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: new Date(), + ...patch, + }; + if (waitCondition && issueId && isUuidLike(issueId)) { + const waitIssueId = issueId; + return db.transaction(async (tx) => { + await tx.execute(sql`select id from issues where id = ${waitIssueId} and company_id = ${agent.companyId} for update`); + if (durableRequest) await notifyChatInboundWakeNotices(tx, { + companyId: durableRequest.companyId, issueId: durableRequest.issueId, + }); + return recordExecutionWait(tx as unknown as Db, { + issueId: waitIssueId, request, condition: waitCondition, coalesce: coalesceExecutionWait, + }); + }); + } + if (durableRequest) { + await db.transaction(async tx => { + await notifyChatInboundWakeNotices(tx, { + companyId: durableRequest.companyId, issueId: durableRequest.issueId, + }); + await tx.insert(agentWakeupRequests).values(request); + }); + } else { + await db.insert(agentWakeupRequests).values(request); + } + return { created: true }; + }; + const writeSkippedHeartbeatRequest = async ( + skipReason: string, + details: Record, + ) => { + await writeSkippedRequest(skipReason, { + payload: { + ...(payload ?? {}), + heartbeatSkip: details, + }, + }); + }; + + const schedulingSuppression = await getSchedulingSuppression(); + // A task drain holds ADMISSION, not the request. The drain is a + // process-local pre-restart hold, so a wake that arrives while it is + // active still names real work that must run once the process comes + // back: leave it in the durable queue and let the dispatch-side checks + // (startNextQueuedRunForAgent / executeRun) keep it from starting until + // the drain lifts or the restart clears it. Writing it as `skipped` + // here dropped the wake permanently — an accepted plan whose + // continuation wake landed mid-drain left its issue in `todo` with no + // run and no path until a person noticed. + if ( + schedulingSuppression.suppressed && + schedulingSuppression.reason !== "task_drain" + ) { + await writeSkippedHeartbeatRequest("heartbeat.scheduling_suppressed", { + reason: schedulingSuppression.reason, + }); + return null; + } + + const worktreeExecutionCutoff = + opts.requestedByActorType === "user" + ? null + : await getWorktreeExecutionCutoff(); + + const company = await db + .select({ status: companies.status }) + .from(companies) + .where(eq(companies.id, agent.companyId)) + .then((rows) => rows[0] ?? null); + + if (!company || company.status !== "active") { + if (caseRequest) return null; + const companyStatus = company?.status ?? "missing"; + if (opts.requestedByActorType === "user") { + throw conflict("Company is not active", { status: companyStatus }); + } + await writeSkippedRequest("company.inactive", { + error: `Wake suppressed because company status is ${companyStatus}`, + }, { companyStatus }); + return null; + } + + const explicitResumeSession = await resolveExplicitResumeSessionOverride( + agent, + payload, + taskKey, + ); + if (explicitResumeSession) { + enrichedContextSnapshot.resumeFromRunId = + explicitResumeSession.resumeFromRunId; + enrichedContextSnapshot.resumeSessionDisplayId = + explicitResumeSession.sessionDisplayId; + enrichedContextSnapshot.resumeSessionParams = + explicitResumeSession.sessionParams; + if ( + !readNonEmptyString(enrichedContextSnapshot.issueId) && + explicitResumeSession.issueId + ) { + enrichedContextSnapshot.issueId = explicitResumeSession.issueId; + } + if ( + !readNonEmptyString(enrichedContextSnapshot.taskId) && + explicitResumeSession.taskId + ) { + enrichedContextSnapshot.taskId = explicitResumeSession.taskId; + } + if ( + !readNonEmptyString(enrichedContextSnapshot.taskKey) && + explicitResumeSession.taskKey + ) { + enrichedContextSnapshot.taskKey = explicitResumeSession.taskKey; + } + issueId = readNonEmptyString(enrichedContextSnapshot.issueId) ?? issueId; + } + const effectiveTaskKey = + readNonEmptyString(enrichedContextSnapshot.taskKey) ?? taskKey; + const genericTimerWake = + source === "timer" && + !issueId && + !wakeCommentId && + !readNonEmptyString(enrichedContextSnapshot.taskId) && + !effectiveTaskKey; + // Old or direct timer producers cannot create a doomed taskless Runner + // run. Explicit session restoration above can supply a real task scope. + if (genericTimerWake && !supportsUnscopedTimerHeartbeat(agent)) return null; + const sessionBefore = + explicitResumeSession?.sessionDisplayId ?? + (await resolveSessionBeforeForWakeup(agent, effectiveTaskKey)); + let hasResolvablePriorSessionWorkspace: boolean | null = null; + const resolveHasResolvablePriorSessionWorkspace = async () => { + if (hasResolvablePriorSessionWorkspace !== null) + return hasResolvablePriorSessionWorkspace; + hasResolvablePriorSessionWorkspace = issueId + ? await hasResolvablePriorSessionWorkspaceForWake({ + agent, + contextSnapshot: enrichedContextSnapshot, + taskKey: effectiveTaskKey, + explicitResumeSession, + }) + : false; + return hasResolvablePriorSessionWorkspace; + }; + const continuationAttempt = readContinuationAttempt( + enrichedContextSnapshot.livenessContinuationAttempt, + ); + + let projectId = readNonEmptyString(enrichedContextSnapshot.projectId); + if (!projectId && issueId) { + // Look up by either UUID or identifier (e.g. "ENV-13"), but always scope + // by companyId so a row from another tenant can never be returned even + // when identifiers collide across companies. Guard the UUID arm because + // issues.id is a Postgres uuid column — passing "ENV-13" into eq(issues.id, …) + // would fail with an invalid-input-syntax cast error before the OR is + // evaluated. + const lookupIsUuid = isUuidLike(issueId); + const idMatch = lookupIsUuid + ? or( + eq(issues.id, issueId), + eq(issues.identifier, issueId.toUpperCase()), + ) + : eq(issues.identifier, issueId.toUpperCase()); + const resolvedIssue = await db + .select({ + id: issues.id, + projectId: issues.projectId, + createdAt: issues.createdAt, + }) + .from(issues) + .where(and(eq(issues.companyId, agent.companyId), idMatch)) + .then((rows) => rows[0] ?? null); + if (resolvedIssue) { + if ( + worktreeExecutionCutoff && + resolvedIssue.createdAt < worktreeExecutionCutoff + ) { + await writeSkippedHeartbeatRequest( + "heartbeat.worktree_execution_cutoff", + { + reason: "worktree_execution_cutoff", + cutoff: worktreeExecutionCutoff.toISOString(), + issueId: resolvedIssue.id, + }, + ); + return null; + } + projectId = resolvedIssue.projectId ?? null; + // Canonicalize context to the UUID so downstream lookups always use UUID + if (resolvedIssue.id !== issueId) { + issueId = resolvedIssue.id; + enrichedContextSnapshot.issueId = issueId; + if (readNonEmptyString(enrichedContextSnapshot.taskId)) { + enrichedContextSnapshot.taskId = issueId; + } + } + } + } + // Propagate projectId into context so resolveWorkspaceForRun can bind the + // project workspace even when context.projectId wasn't set by the caller. + if (projectId && !readNonEmptyString(enrichedContextSnapshot.projectId)) { + enrichedContextSnapshot.projectId = projectId; + } + const isolatedWorkspacesEnabled = issueId + ? (await instanceSettings.getExperimental()).enableIsolatedWorkspaces + : false; + let operatorResponsibleUserId: string | null = opts.manualUserWake ? opts.requestedByActorId! : null; + let queuedResponsibleUserIdPromise: Promise | null = null; + const resolveQueuedResponsibleUserId = () => { + if (operatorResponsibleUserId) return Promise.resolve(operatorResponsibleUserId); + queuedResponsibleUserIdPromise ??= (async () => { + const queuedIssueContext = issueId + ? await getIssueExecutionContext(agent.companyId, issueId) + : null; + const queuedRoutineEnvContext = await getRoutineEnvForExecutionIssue( + agent.companyId, + queuedIssueContext, + ); + const queuedResponsibleUserId = + await resolveResponsibleUserIdForRunSeed({ + companyId: agent.companyId, + contextSnapshot: enrichedContextSnapshot, + issueContext: queuedIssueContext, + routineEnvContext: queuedRoutineEnvContext, + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + source, + triggerDetail, + }); + if (!queuedResponsibleUserId) { + throw new HttpError( + 422, + "Unable to resolve responsible user for heartbeat run dispatch", + { + code: "responsible_user_unresolved", + agentId, + companyId: agent.companyId, + issueId: issueId ?? null, + source, + triggerDetail, + wakeReason: readNonEmptyString( + enrichedContextSnapshot.wakeReason, + ), + }, + ); + } + return queuedResponsibleUserId; + })(); + return queuedResponsibleUserIdPromise; + }; + + const budgetBlock = await budgets.getInvocationBlock( + agent.companyId, + agentId, + { + issueId, + projectId, + }, + ); + if (budgetBlock) { + await writeSkippedRequest("budget.blocked", { error: budgetBlock.reason }, { + scopeType: budgetBlock.scopeType, scopeId: budgetBlock.scopeId, + }); + if (caseRequest) return null; + throw conflict(budgetBlock.reason, { + scopeType: budgetBlock.scopeType, + scopeId: budgetBlock.scopeId, + }); + } + + // Setup delays execution, but must not discard work accepted by a caller. + const invokability = await getAgentInvokability(isAgentAwaitingSetup(agent) ? { ...agent, status: "idle" } : agent); + if (!invokability.invokable) { + if (caseRequest) return null; + if (opts.requestedByActorType !== "user" || executionWaitRequestId) { + await writeSkippedRequest("agent.not_invokable", { + error: invokability.message, + }, { status: agent.status, reason: invokability.reason }); + } + throw conflict(invokability.message, { + status: agent.status, + reason: invokability.reason, + invalidOrgChain: invokability.invalidOrgChain, + ...invokability.details, + }); + } + + const policy = parseHeartbeatPolicy(agent); + + if (source === "timer" && !policy.enabled) { + await writeSkippedRequest("heartbeat.disabled", {}, { enabled: false }); + return null; + } + if (source !== "timer" && !policy.wakeOnDemand) { + await writeSkippedRequest("heartbeat.wakeOnDemand.disabled", {}, { wakeOnDemand: false }); + return null; + } + + if ( + policy.skipTimerWhenNoActionableWork && + genericTimerWake && + !(await hasActionableTimerWork(agent)) + ) { + await writeSkippedHeartbeatRequest("heartbeat.timer.no_actionable_work", { + reason: + "No assigned todo or in_progress issue requires this agent before timer adapter invocation.", + }); + await markTimerHeartbeatChecked(agentId, source); + return null; + } + + if (issueId) { + const activePauseHold = await treeControlSvc.getActivePauseHoldGate( + agent.companyId, + issueId, + ); + if (activePauseHold) { + const treeHoldInteractionWake = + await isVerifiedIssueTreeControlInteractionWake(db, { + companyId: agent.companyId, + issueId, + agentId, + contextSnapshot: enrichedContextSnapshot, + requestedByActorType: opts.requestedByActorType, + requestedByActorId: opts.requestedByActorId, + }); + + if (!treeHoldInteractionWake) { + const wait = await writeSkippedRequest("issue_tree_hold_active", {}, { + holdId: activePauseHold.holdId, + }); + if (wait.created) await logActivity(db, { + companyId: agent.companyId, + actorType: "system", + actorId: "system", + agentId, + runId: null, + action: "issue.tree_hold_wakeup_deferred", + entityType: "issue", + entityId: issueId, + details: { + holdId: activePauseHold.holdId, + rootIssueId: activePauseHold.rootIssueId, + requestedReason: reason, + source, + triggerDetail, + securityPrinciples: [ + "Complete Mediation", + "Fail Securely", + "Secure Defaults", + ], + }, + }); + return null; + } + + enrichedContextSnapshot.treeHoldInteraction = true; + enrichedContextSnapshot.activeTreeHold = { + holdId: activePauseHold.holdId, + rootIssueId: activePauseHold.rootIssueId, + mode: activePauseHold.mode, + reason: activePauseHold.reason, + releasePolicy: activePauseHold.releasePolicy, + interaction: true, + }; + } + } + + if (issueId) { + // Mention-triggered wakes can request input from another agent, but they must + // still respect the issue execution lock so a second agent cannot start on the + // same issue workspace while the assignee already has a live run. + const agentNameKey = normalizeAgentNameKey(agent.name); + + const cancelledRunsToEmit: (typeof heartbeatRuns.$inferSelect)[] = []; + + const outcome = await db + .transaction(withDurableChatAdmission(durableRequest, async (tx) => { + await tx.execute( + sql`select id from issues where id = ${issueId} and company_id = ${agent.companyId} for update`, + ); + + if (executionWaitRequestId) { + const [pending] = await tx.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.id, executionWaitRequestId), eq(agentWakeupRequests.companyId, agent.companyId), + eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), + // A user message can join a queue originally created by a + // system wake. Admission validates the saved user comment or board click. + (opts.queuedCommentInterruptId ?? opts.queuedCommentRequestId) === executionWaitRequestId + ? undefined : eq(agentWakeupRequests.requestedByActorType, + durableRequest ? opts.requestedByActorType ?? "" : "user"), + opts.queuedCommentInterruptId === executionWaitRequestId + ? sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt'->>'actorId' = ${opts.requestedByActorId ?? ""}` + : opts.queuedCommentRequestId === executionWaitRequestId ? undefined + : eq(agentWakeupRequests.requestedByActorId, opts.requestedByActorId ?? ""), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, + )); + // The issue lock serializes cleanup callbacks and periodic workers. + // An adopted, discarded, or edited receipt is no longer authority. + if (!pending || (!opts.taskRetryRequestId && !(wakeCommentId && queuedCommentIdsFromWakePayload(pending.payload).includes(wakeCommentId)) && + !(opts.queuedCommentInterruptId && await readQueuedInteractionResponse(tx as unknown as Db, + agent.companyId, issueId, pending.payload)))) { + return { kind: "deferred" as const }; + } + if (opts.taskRetryRequestId && durableRequest?.failedRunRetry) { + if (pending.id !== durableRequest.id || pending.payload?.taskRetryRequestId !== opts.taskRetryRequestId) + return { kind: "deferred" as const }; + // The parent board Retry is checked separately below after the + // connected conversation reauthorizes this exact child receipt. + } else if (opts.taskRetryRequestId) { + const retry = readTaskRetryIntent(pending.payload); + if (pending.id !== opts.taskRetryRequestId || retry?.generationId !== pending.id || retry.chatActionId || + retry.actorId !== opts.requestedByActorId || retry.sourceRunId !== enrichedContextSnapshot.previousRunId) + return { kind: "deferred" as const }; + if (!await taskRetryMutationAllowed(tx as unknown as Db, { companyId: agent.companyId, issueId, + actorId: retry.actorId, actorSource: retry.actorSource })) return { kind: "deferred" as const }; + if (retry.boardMessageOrigin) { + const origin = await readTaskRetryBoardMessageOrigin(tx as unknown as Db, { companyId: agent.companyId, + issueId, agentId, sourceRunId: retry.sourceRunId, actorId: retry.actorId }); + if (!origin || origin.commentId !== retry.boardMessageOrigin.commentId || + origin.identityContextId !== retry.boardMessageOrigin.identityContextId) return { kind: "deferred" as const }; + } + } + if (opts.queuedCommentRequestId) { + const ids = await undeliveredLegacyUserCommentIds(tx as unknown as Db, + agent.companyId, issueId, agentId, queuedCommentIdsFromWakePayload(pending.payload)); + if (!wakeCommentId || !ids.includes(wakeCommentId)) return { kind: "deferred" as const }; + pending.payload = withQueuedCommentIdsInWakePayload(parseObject(pending.payload), ids); + await tx.update(agentWakeupRequests).set({ payload: pending.payload }).where(and( + eq(agentWakeupRequests.id, pending.id), eq(agentWakeupRequests.companyId, agent.companyId), + )); + } + if (!opts.queuedCommentInterruptId && !opts.queuedCommentRequestId && pending.payload?.manualUserWake === true) { + // A persisted manual wake keeps its actor when an execution wait + // resumes. The locked receipt above has revalidated that actor. + payload = { ...payload, manualUserWake: true }; + operatorResponsibleUserId = opts.requestedByActorId!; + } + if (opts.queuedCommentInterruptId) { + // The locked board receipt supplies execution authority even when + // another user authored the messages. Dispatch revalidates the receipt. + operatorResponsibleUserId = opts.requestedByActorId!; + } + if (opts.queuedCommentInterruptId || opts.queuedCommentRequestId) { + // Edits/discards between the click and dispatch remain authoritative. + Object.assign(enrichedContextSnapshot, withQueuedCommentIdsInRunContext( + enrichedContextSnapshot, queuedCommentIdsFromWakePayload(pending.payload), + )); + } + } + let automaticParentRunId: string | null = null; + if ( + source === "automation" && + opts.requestedByActorType === "system" + ) { + const nativeParent = await readChatControlNativeParent( + tx as unknown as Db, + { companyId: agent.companyId, issueId, agentId }, + opts.requestedByActorId ?? null, + ); + const genericParent = + !opts.requestedByActorId && reason === "issue_continuation_needed" + ? readNonEmptyString(enrichedContextSnapshot.retryOfRunId) + : null; + automaticParentRunId = + nativeParent.kind === "parent" + ? nativeParent.runId + : genericParent; + if (automaticParentRunId) { + const [parent] = await tx + .select({ + agentId: heartbeatRuns.agentId, + context: heartbeatRuns.contextSnapshot, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, agent.companyId), + eq(heartbeatRuns.id, automaticParentRunId), + ), + ) + .limit(1); + if ( + parent && + (parseObject(parent.context).issueId ?? + parseObject(parent.context).taskId) === issueId && + parent.agentId !== agentId + ) + automaticParentRunId = null; + } + const proof = + nativeParent.kind === "unresolved" + ? { kind: "unresolved" as const } + : automaticParentRunId + ? await readChatControlRecoveryStop( + tx as unknown as Db, + { + companyId: agent.companyId, + issueId, + agentId, + sourceRunId: automaticParentRunId, + }, + true, + ) + : { kind: "clear" as const }; + if (proof.kind !== "clear") { + await tx + .insert(agentWakeupRequests) + .values({ + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: + proof.kind === "stopped" + ? CHAT_CONTROL_RECOVERY_STOP_CODE + : CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE, + payload: { issueId }, + status: "skipped", + requestedByActorType: "system", + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: new Date(), + }); + return { kind: "skipped" as const }; + } + } + + const durableReceipt = await existingDurableReceipt( + tx as unknown as Db, + ); + if (durableReceipt) + return { kind: "durable" as const, receipt: durableReceipt }; + if (caseRequest) { + const [handoff] = await tx.select().from(caseHandoffs).where(and(eq(caseHandoffs.id, caseRequest.handoffId), + eq(caseHandoffs.companyId, agent.companyId), + eq(caseHandoffs.issueId, caseRequest.issueId), eq(caseHandoffs.state, "pending_dispatch"), + eq(caseHandoffs.toAgentId, agentId), eq(caseHandoffs.wakeupRequestId, caseRequest.wakeupRequestId))).for("update"); + if (!handoff || handoff.appliedIssueStatusVersion !== caseRequest.expectedIssueStatusVersion) + throw conflict("Case handoff delivery was superseded"); + } + const failedChatRetry = await authorizeFailedChatRunRetryWake( + db, + tx as unknown as Db, + { + phase: "admission", + wakeupRequestId: durableRequest?.id ?? null, + companyId: agent.companyId, + agentId, + issueId, + contextSnapshot: enrichedContextSnapshot, + }, + ); + if ( + failedChatRetry !== Boolean(durableRequest?.failedRunRetry) || + (failedChatRetry && + durableRequest?.failedRunRetry?.failedRunId !== + readNonEmptyString(enrichedContextSnapshot.retryOfRunId)) + ) { + throw new FailedChatRunRetryAuthorizationError(); + } + if (durableRequest) { + if (issueId !== durableRequest.issueId) + throw new Error("chat_inbound_wakeup_binding_denied"); + await durableRequest.authorize(tx as unknown as Db); + if (opts.taskRetryRequestId) { + const [retryWake] = await tx.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.id, opts.taskRetryRequestId), + eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), + )).for("update"); + const retry = readTaskRetryIntent(retryWake?.payload); + if (!failedChatRetry || retry?.chatActionId !== durableRequest.id || + retry.actorId !== taskRetryActorId || retry.sourceRunId !== durableRequest.failedRunRetry?.failedRunId) + throw new FailedChatRunRetryAuthorizationError(); + if (!await taskRetryMutationAllowed(tx as unknown as Db, { companyId: agent.companyId, issueId, + actorId: retry.actorId, actorSource: retry.actorSource })) return { kind: "deferred" as const }; + } + } else { + if ( + source === "on_demand" && + triggerDetail === "manual" && + reason === "retry_failed_run" + ) { + // Generic Board retry does not carry server-authorized failed-run + // lineage. Caller-supplied source/comment/retry markers cannot + // restore it, including for a retired conversation generation. + const [chatBinding] = await tx + .select({ id: chatConversations.id }) + .from(chatConversations) + .innerJoin(chatEndpoints, eq(chatEndpoints.id, chatConversations.endpointId)) + .where( + and( + eq(chatEndpoints.externalExecutionPolicy, "restricted"), + eq(chatConversations.companyId, agent.companyId), + eq(chatConversations.issueId, issueId), + ), + ) + .limit(1); + if (chatBinding) { + throw conflict( + "Retry needs the exact failed chat request and current access. Send the request again in the current connected conversation.", + { code: "chat_failed_run_retry_requires_authorized_context" }, + ); + } + } + const [held] = await tx + .select({ id: issues.id }) + .from(issues) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, agent.companyId), + unadmittedChatWakeupCondition(issues.id, issues.companyId), + ), + ) + .limit(1); + if (held) { + throw conflict( + "This task has external chat input that has not been admitted. Check the connection's Activity; let pending input finish, or restore access and send a new authorized message before starting the task.", + { code: "chat_inbound_wakeup_unadmitted", issueId }, + ); + } + } + + const issue = await tx + .select({ + id: issues.id, + companyId: issues.companyId, + identifier: issues.identifier, + conversationAgentId: issues.conversationAgentId, + conversationUserId: issues.conversationUserId, + conversationState: issues.conversationState, + status: issues.status, + statusVersion: issues.statusVersion, + projectId: issues.projectId, + projectWorkspaceId: issues.projectWorkspaceId, + executionWorkspaceId: issues.executionWorkspaceId, + executionWorkspacePreference: issues.executionWorkspacePreference, + executionWorkspaceSettings: issues.executionWorkspaceSettings, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + monitorNextCheckAt: issues.monitorNextCheckAt, + monitorWakeRequestedAt: issues.monitorWakeRequestedAt, + executionRunId: issues.executionRunId, + executionAgentNameKey: issues.executionAgentNameKey, + createdAt: issues.createdAt, + }) + .from(issues) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, agent.companyId), + ), + ) + .then((rows) => rows[0] ?? null); + + if (!issue) { + if (caseRequest) throw conflict("Case handoff task no longer exists"); + await tx.insert(agentWakeupRequests).values({ + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: "issue_execution_issue_not_found", + payload, + status: "skipped", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: new Date(), + }); + return { kind: "skipped" as const }; + } + + if (taskRetryActorId) { + await assertTaskRetryConversationAllowed(tx as unknown as Db, issue, taskRetryActorId); + if (await retireSupersededTaskRetry(tx as unknown as Db, { + companyId: issue.companyId, issueId: issue.id, requestId: opts.taskRetryRequestId!, + sourceRunId: readNonEmptyString(enrichedContextSnapshot.previousRunId)!, + })) return { kind: "deferred" as const }; + } + const issueStateGuard = opts.issueStateGuard; + const activeMonitorRun = issueStateGuard?.monitorNextCheckAt === undefined ? null + : await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, issue.companyId), eq(heartbeatRuns.nativeIssueId, issue.id), + eq(heartbeatRuns.runtimeMode, "native"), inArray(heartbeatRuns.status, ["queued", "running", "scheduled_retry"]), + )).limit(1).then(rows => rows[0] ?? null); + if ( + issueStateGuard && + (!issueStateGuard.statuses.includes(issue.status) || + issue.assigneeAgentId !== issueStateGuard.assigneeAgentId || + (issueStateGuard.statusVersion !== undefined && issue.statusVersion !== issueStateGuard.statusVersion) || + (issueStateGuard.monitorNextCheckAt !== undefined && ( + activeMonitorRun !== null || issue.assigneeUserId !== null || + issue.monitorNextCheckAt?.toISOString() !== issueStateGuard.monitorNextCheckAt || + issue.monitorWakeRequestedAt?.toISOString() !== issueStateGuard.monitorWakeRequestedAt + ))) + ) { + if (caseRequest) throw conflict("Case handoff task changed before admission"); + // A deferred monitor retains its schedule; do not create a receipt + // that could suppress its next admission attempt. + if (issueStateGuard.monitorNextCheckAt !== undefined) return { kind: "skipped" as const }; + await tx.insert(agentWakeupRequests).values({ + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: "issue_state_guard_mismatch", + payload: { + ...(payload ?? {}), + heartbeatSkip: { + reason: + "Issue status or assignee changed before the wake could be queued.", + issueId: issue.id, + expectedStatuses: issueStateGuard.statuses, + actualStatus: issue.status, + expectedAssigneeAgentId: issueStateGuard.assigneeAgentId, + actualAssigneeAgentId: issue.assigneeAgentId, + }, + }, + status: "skipped", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: new Date(), + }); + return { kind: "skipped" as const }; + } + + if (opts.failedRunId) { + // The issue lock makes double-clicks and network retries adopt the + // same successor, including after it has already finished. + const [previousRetry] = await tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, issue.companyId), eq(heartbeatRuns.agentId, agentId), + eq(heartbeatRuns.retryOfRunId, opts.failedRunId), + sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'retry_failed_run'`, + )).orderBy(desc(heartbeatRuns.createdAt)).limit(1); + if (previousRetry) return { kind: "replayed" as const, run: previousRetry }; + } + + let reconciledSourceRunId: string | null = null; + let reconciledRestoreRetryCount: number | null = null; + if (executionReconciliationWake) { + const actionId = readNonEmptyString( + enrichedContextSnapshot.recoveryActionId, + ); + if ( + !actionId || + !isUuidLike(actionId) || + source !== "automation" || + triggerDetail !== "system" || + reason !== "issue_recovery_action_restored" || + opts.requestedByActorType !== "system" || + opts.requestedByActorId !== "execution-recovery" || + opts.idempotencyKey !== `execution-reconciliation:${actionId}` || + enrichedContextSnapshot.source !== "execution.reconciled" || + enrichedContextSnapshot.forceFreshSession !== true || + payload?.issueId !== issue.id || + payload?.recoveryActionId !== actionId || + issue.assigneeAgentId !== agentId || + ["done", "cancelled"].includes(issue.status) + ) + return { kind: "skipped" as const }; + + // The issue lock serializes all admissions for this source. Validate + // the durable operator decision, then reconcile a prior queue commit + // before considering a new wake (including a now-terminal successor). + const [action] = await tx + .select() + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, issue.companyId), + eq(issueRecoveryActions.sourceIssueId, issue.id), + eq(issueRecoveryActions.id, actionId), + ), + ) + .for("update"); + const decision = parseObject( + action?.evidence.executionReconciliation, + ); + const sourceRunId = readNonEmptyString(decision.runId); + if ( + !action || + action.status !== "resolved" || + action.kind !== "active_run_watchdog" || + action.returnOwnerAgentId !== agentId || + !sourceRunId || + !isUuidLike(sourceRunId) || + decision.providerStopped !== true || + !["completed", "not_performed", "mixed"].includes( + String(decision.actionOutcome), + ) || + !readNonEmptyString(decision.outcomeEvidence) || + enrichedContextSnapshot.previousRunId !== sourceRunId || + enrichedContextSnapshot.retryOfRunId !== sourceRunId || + !["pending", "delivered"].includes( + String(action.evidence.continuationDelivery), + ) + ) + return { kind: "skipped" as const }; + + const [existingWake] = await tx + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, issue.companyId), + eq(agentWakeupRequests.agentId, agentId), + eq(agentWakeupRequests.idempotencyKey, opts.idempotencyKey), + ne(agentWakeupRequests.status, "skipped"), + ), + ) + .orderBy(asc(agentWakeupRequests.requestedAt)) + .limit(1); + if (existingWake) { + if ( + existingWake.payload?.issueId !== issue.id || + existingWake.payload?.recoveryActionId !== action.id || + existingWake.requestedByActorType !== "system" || + existingWake.requestedByActorId !== "execution-recovery" || + !existingWake.runId + ) + return { kind: "deferred" as const }; + const [existingRun] = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, issue.companyId), + eq(heartbeatRuns.agentId, agentId), + eq(heartbeatRuns.id, existingWake.runId), + ), + ); + if ( + !existingRun || + existingRun.contextSnapshot?.issueId !== issue.id || + existingRun.contextSnapshot?.recoveryActionId !== action.id || + existingRun.contextSnapshot?.previousRunId !== sourceRunId + ) + return { kind: "deferred" as const }; + return { kind: "replayed" as const, run: existingRun }; + } + if (action.evidence.continuationDelivery !== "pending") + return { kind: "skipped" as const }; + const [reconciledRun] = await tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, issue.companyId), eq(heartbeatRuns.id, sourceRunId), + )); + if (decision.workspaceReconstruction) { + await validateNativeWorkspaceReconstruction(tx as unknown as Db, { + companyId: issue.companyId, issueId: issue.id, runId: sourceRunId, actionId, + }); + reconciledRestoreRetryCount = executionFailureRetryCount(reconciledRun!); + enrichedContextSnapshot.workspaceReconstructionActionId = action.id; + } + if (hasWorkspaceRestoreFailure(reconciledRun?.resultJson)) { + if ((readNonEmptyString(decision.workspaceRepairEvidence)?.length ?? 0) < 20) + return { kind: "skipped" as const }; + // Repair does not reset the remaining automatic retry budget. + reconciledRestoreRetryCount = executionFailureRetryCount(reconciledRun!); + } + reconciledSourceRunId = sourceRunId; + } + + let continuationWait = { reason: "execution_recovery", message: "Waiting for execution recovery. Your message is saved." }; + let continuationBlocked = false; + const deferBlockedExecution = async ( + executionBlocker: Pick>>, "recoveryActionId" | "cause" | "nextAction">, + ) => { + if (caseRequest) waitForCaseAdmission("execution_reconciliation_required", executionBlocker.nextAction); + opts.issueStateGuard?.onExecutionDeferred?.(executionBlocker.cause); + const condition = { recoveryActionId: executionBlocker.recoveryActionId, ...continuationWait }; + if (executionWaitRequestId) { + await tx.update(agentWakeupRequests).set({ + payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', ${JSON.stringify(condition)}::jsonb)`, + updatedAt: new Date(), + }).where(eq(agentWakeupRequests.id, executionWaitRequestId)); + return { kind: "deferred" as const, admitted: true }; + } + if (durableRequest || wakeCommentId || + hasInteractionContinuationWakeContext(enrichedContextSnapshot) || + readNonEmptyString(enrichedContextSnapshot.nativeStatusWakeIntentId)) { + await tx.insert(agentWakeupRequests).values({ + ...durableReceiptFields, + companyId: agent.companyId, agentId, source, triggerDetail, reason, + payload: withQueuedCommentIdsInWakePayload({ + ...payload, + issueId: issue.id, + [DEFERRED_WAKE_CONTEXT_KEY]: enrichedContextSnapshot, + executionWait: condition, + }, [...new Set([ + ...queuedCommentIdsFromRunContext(enrichedContextSnapshot), + ...(wakeCommentId ? [wakeCommentId] : []), + ])]), + status: "deferred_issue_execution", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + }); + } else { + await recordExecutionWait(tx as unknown as Db, { + issueId: issue.id, condition, coalesce: coalesceExecutionWait, + request: { + ...durableReceiptFields, + companyId: agent.companyId, agentId, source, triggerDetail, + reason: "execution_reconciliation_required", + error: executionBlocker.nextAction, + payload, + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + }, + }); + } + return { kind: "deferred" as const, admitted: true }; + }; + const explicitContinuationRunId = randomUUID(); + const [resetComment] = isConversation(issue) && opts.requestedByActorType === "user" && opts.requestedByActorId && wakeCommentId + ? await tx.select({ body: issueComments.body }).from(issueComments).where(and( + eq(issueComments.companyId, issue.companyId), eq(issueComments.issueId, issue.id), + eq(issueComments.id, wakeCommentId), eq(issueComments.authorUserId, opts.requestedByActorId), + eq(issueComments.authorType, "user"), isNull(issueComments.deletedAt), isNull(issueComments.createdByRunId), + )).limit(1) + : []; + const verifiedConversationReset = Boolean(resetComment && isConversationReset(resetComment.body)); + const executionBlocker = await getExecutionBlocker( + tx as unknown as Db, issue.companyId, issue.id, + { conversationResetCommentId: verifiedConversationReset ? wakeCommentId : null }, + ); + // Prove eligibility without retiring the hold. Later gates can still + // decline this wake; hold retirement and successor creation stay atomic. + // A bound chat request has already rechecked its current principal + // above. Treat its new user message like a board comment, but keep + // failed-run retry actions on their separate exact-request path. + const ownsReconstructionContinuation = reconciledSourceRunId && executionBlocker?.runId === reconciledSourceRunId + && executionBlocker.recoveryActionId === enrichedContextSnapshot.recoveryActionId; + const continuationHold = () => executionBlocker ?? { + recoveryActionId: null, cause: "native_continuation_requires_reconciliation" as const, nextAction: continuationWait.message, + }; + if ((executionBlocker || opts.queuedCommentInterruptId) && !ownsReconstructionContinuation && !(await admitExplicitNativeContinuation({ + db: tx as unknown as Db, companyId: issue.companyId, issueId: issue.id, + agentId, actorType: taskRetryActorId ? "user" : opts.requestedByActorType, actorId: taskRetryActorId ?? opts.requestedByActorId, + reason: taskRetryActorId ? "task_retry" : durableRequest && !failedChatRetry ? "issue_commented" : reason, + commentId: wakeCommentId ?? null, failedRunId: opts.failedRunId, successorRunId: explicitContinuationRunId, + queuedCommentInterruptId: opts.queuedCommentInterruptId, + queuedCommentRequestId: opts.queuedCommentRequestId, + taskRetryRequestId: opts.taskRetryRequestId, + ...(durableRequest?.resumeDeferred && !failedChatRetry ? { durableChatRequest: durableRequest } : {}), + dryRun: true, + onBlocked: (reason, message) => { continuationBlocked = true; continuationWait = { reason, message }; }, + })) && (executionBlocker || (continuationBlocked && !verifiedConversationReset))) return deferBlockedExecution(continuationHold()); + + + if ( + worktreeExecutionCutoff && + issue.createdAt < worktreeExecutionCutoff + ) { + if (caseRequest) waitForCaseAdmission("heartbeat.worktree_execution_cutoff", "Case handoff is waiting for workspace execution to be enabled"); + await tx.insert(agentWakeupRequests).values({ + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: "heartbeat.worktree_execution_cutoff", + payload: { + ...(payload ?? {}), + heartbeatSkip: { + reason: "worktree_execution_cutoff", + cutoff: worktreeExecutionCutoff.toISOString(), + issueId: issue.id, + }, + }, + status: "skipped", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: new Date(), + }); + return { kind: "skipped" as const }; + } + + const cancelStaleScheduledRetry = async ( + scheduledRun: typeof heartbeatRuns.$inferSelect, + ) => { + const issueCancelled = issue.status === "cancelled"; + if ( + scheduledRun.status !== "scheduled_retry" || + (scheduledRun.agentId === issue.assigneeAgentId && + !issueCancelled) + ) { + return false; + } + + const now = new Date(); + const reason = issueCancelled + ? "Cancelled because the issue was cancelled before the scheduled retry became due" + : "Cancelled because the issue was reassigned before the scheduled retry became due"; + await notifyInstructionCollection(tx, { companyId: scheduledRun.companyId, runId: scheduledRun.id }); + await notifyExternalAgentSettlement(tx, { companyId: scheduledRun.companyId, runId: scheduledRun.id }); + await notifyCancelledWorkspacePreservation(tx, { companyId: scheduledRun.companyId, runId: scheduledRun.id }); + await notifyUnsafeWorkspaceExportRunChange(tx, scheduledRun); + await notifyScheduledRetryCancellation(tx, { companyId: scheduledRun.companyId, runId: scheduledRun.id }); + await notifyChatRunMilestones(tx, { companyId: scheduledRun.companyId, issueId: issue.id }); + const cancelled = await tx + .update(heartbeatRuns) + .set({ + status: "cancelled", + finishedAt: now, + error: reason, + errorCode: issueCancelled + ? "issue_cancelled" + : "issue_reassigned", + updatedAt: now, + }) + .where( + and( + eq(heartbeatRuns.id, scheduledRun.id), + eq(heartbeatRuns.status, "scheduled_retry"), + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + + if (!cancelled) return false; + + if (scheduledRun.wakeupRequestId) { + await tx + .update(agentWakeupRequests) + .set({ + status: "cancelled", + finishedAt: now, + error: reason, + updatedAt: now, + }) + .where( + eq(agentWakeupRequests.id, scheduledRun.wakeupRequestId), + ); + } + + if (issue.executionRunId === scheduledRun.id) { + await tx + .update(issues) + .set({ + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: now, + }) + .where( + and( + eq(issues.id, issue.id), + eq(issues.executionRunId, scheduledRun.id), + ), + ); + } + + const eventSeq = await allocateHeartbeatRunEventSeq( + tx as unknown as Db, + cancelled.id, + ); + + await tx.insert(heartbeatRunEvents).values({ + companyId: cancelled.companyId, + runId: cancelled.id, + agentId: cancelled.agentId, + seq: eventSeq, + eventType: "lifecycle", + stream: "system", + level: "warn", + message: issueCancelled + ? "Scheduled retry cancelled because issue was cancelled before it became due" + : "Scheduled retry cancelled because issue ownership changed before it became due", + payload: { + issueId: issue.id, + issueStatus: issue.status, + scheduledRetryAttempt: cancelled.scheduledRetryAttempt, + scheduledRetryAt: cancelled.scheduledRetryAt + ? new Date(cancelled.scheduledRetryAt).toISOString() + : null, + scheduledRetryReason: cancelled.scheduledRetryReason, + previousRetryAgentId: cancelled.agentId, + currentAssigneeAgentId: issue.assigneeAgentId, + }, + }); + await tx + .update(heartbeatRuns) + .set({ nextEventSeq: eventSeq + 1, updatedAt: now }) + .where(eq(heartbeatRuns.id, cancelled.id)); + + cancelledRunsToEmit.push(cancelled); + + return true; + }; + + let activeExecutionRun = issue.executionRunId + ? await tx + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, issue.executionRunId)) + .then((rows) => rows[0] ?? null) + : null; + + if ( + activeExecutionRun && + !EXECUTION_PATH_HEARTBEAT_RUN_STATUSES.includes( + activeExecutionRun.status as (typeof EXECUTION_PATH_HEARTBEAT_RUN_STATUSES)[number], + ) + ) { + activeExecutionRun = null; + } + + if ( + activeExecutionRun && + (await cancelStaleScheduledRetry(activeExecutionRun)) + ) { + activeExecutionRun = null; + } + + // A queued/scheduled run holding the lock for an agent that is + // no longer the issue's assignee is stale by design — the issue + // has been re-routed (e.g. blocked → in_review with a different + // assignee). Cancel it and release the lock; otherwise the new + // assignee's wake gets parked in `deferred_issue_execution` + // forever, because the original queued holder will never run + // (the issue's status / target now belongs to someone else). + // + // Race guard: pin the cancel UPDATE to the exact non-running + // status we read above. A worker could transition the holder + // from `queued` → `running` between the SELECT and this UPDATE; + // the status predicate ensures we never clobber a freshly- + // claimed running run. If zero rows matched, leave + // `activeExecutionRun` populated so the defer path runs + // normally against the now-running holder. + if ( + activeExecutionRun && + activeExecutionRun.status !== "running" && + issue.assigneeAgentId && + activeExecutionRun.agentId !== issue.assigneeAgentId + ) { + await notifyInstructionCollection(tx, { companyId: activeExecutionRun.companyId, runId: activeExecutionRun.id }); + await notifyExternalAgentSettlement(tx, { companyId: activeExecutionRun.companyId, runId: activeExecutionRun.id }); + await notifyCancelledWorkspacePreservation(tx, { companyId: activeExecutionRun.companyId, runId: activeExecutionRun.id }); + await notifyUnsafeWorkspaceExportRunChange(tx, activeExecutionRun); + await notifyChatRunMilestones(tx, { companyId: activeExecutionRun.companyId, issueId: issue.id }); + const cancelled = await tx + .update(heartbeatRuns) + .set({ + status: "cancelled", + finishedAt: new Date(), + error: + "Execution lock released after issue reassigned to a different agent", + errorCode: "lock_released_on_reassignment", + updatedAt: new Date(), + }) + .where( + and( + eq(heartbeatRuns.id, activeExecutionRun.id), + eq(heartbeatRuns.status, activeExecutionRun.status), + ), + ) + .returning(); + if (cancelled.length > 0) { + cancelledRunsToEmit.push(cancelled[0]); + if (activeExecutionRun.wakeupRequestId) { + await tx + .update(agentWakeupRequests) + .set({ + status: "cancelled", + finishedAt: new Date(), + error: + "Execution lock released after issue reassigned to a different agent", + updatedAt: new Date(), + }) + .where( + eq( + agentWakeupRequests.id, + activeExecutionRun.wakeupRequestId, + ), + ); + } + activeExecutionRun = null; + } + } + + if (!activeExecutionRun && issue.executionRunId) { + await notifyNativeStatusReassessment(tx, { companyId: issue.companyId, issueId: issue.id }); + await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId: issue.id }); + await tx + .update(issues) + .set({ + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: new Date(), + }) + .where(eq(issues.id, issue.id)); + } + + if (!activeExecutionRun) { + const legacyRun = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, issue.companyId), + inArray(heartbeatRuns.status, [ + ...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES, + ]), + sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`, + ), + ) + .orderBy( + sql`case when ${heartbeatRuns.status} = 'running' then 0 else 1 end`, + asc(heartbeatRuns.createdAt), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + + if (legacyRun) { + if (await cancelStaleScheduledRetry(legacyRun)) { + activeExecutionRun = null; + } else { + activeExecutionRun = legacyRun; + const legacyAgent = await tx + .select({ name: agents.name }) + .from(agents) + .where(eq(agents.id, legacyRun.agentId)) + .then((rows) => rows[0] ?? null); + await notifyStatusCardIssueChange(tx, issue.companyId); + await tx + .update(issues) + .set({ + executionRunId: legacyRun.id, + executionAgentNameKey: normalizeAgentNameKey( + legacyAgent?.name, + ), + executionLockedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(issues.id, issue.id)); + } + } + } + + const dependencyReadiness = await issuesSvc + .listDependencyReadiness(issue.companyId, [issue.id], tx) + .then((rows) => rows.get(issue.id) ?? null); + + // Blocked descendants should stay idle until the final blocker resolves. + // Human comment/mention wakes are the exception: they may run in a + // bounded interaction mode so the assignee can answer or triage. + const blockedInteractionWake = + dependencyReadiness && + !dependencyReadiness.isDependencyReady && + allowsIssueInteractionWake( + enrichedContextSnapshot, + ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS, + ); + + if (blockedInteractionWake) { + enrichedContextSnapshot.dependencyBlockedInteraction = true; + enrichedContextSnapshot.unresolvedBlockerIssueIds = + dependencyReadiness.unresolvedBlockerIssueIds; + enrichedContextSnapshot.unresolvedBlockerCount = + dependencyReadiness.unresolvedBlockerCount; + enrichedContextSnapshot.unresolvedBlockerSummaries = + await listUnresolvedBlockerSummaries( + tx, + issue.companyId, + issue.id, + dependencyReadiness.unresolvedBlockerIssueIds, + ); + } + + if ( + !activeExecutionRun && + dependencyReadiness && + !dependencyReadiness.isDependencyReady && + !blockedInteractionWake + ) { + if (caseRequest) waitForCaseAdmission("issue_dependencies_blocked", "Case handoff is waiting for task dependencies to resolve"); + await recordExecutionWait(tx as unknown as Db, { + issueId: issue.id, + coalesce: coalesceExecutionWait, + condition: { unresolvedBlockerIssueIds: [...dependencyReadiness.unresolvedBlockerIssueIds].sort() }, + request: { + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: "issue_dependencies_blocked", + payload: { + ...(payload ?? {}), + issueId, + unresolvedBlockerIssueIds: + dependencyReadiness.unresolvedBlockerIssueIds, + }, + status: "skipped", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: new Date(), + }, + }); + return { kind: "skipped" as const }; + } + + if ( + isolatedWorkspacesEnabled && + !activeExecutionRun && + issue.status !== "done" && + issue.status !== "cancelled" + ) { + const issueSettings = parseIssueExecutionWorkspaceSettings( + issue.executionWorkspaceSettings, + ); + const resolvedMode = resolveExecutionWorkspaceMode({ + projectPolicy: null, + issueSettings, + legacyUseProjectWorkspace: null, + }); + const workspaceManagedConfig = buildExecutionWorkspaceAdapterConfig( + { + agentConfig: parseObject(agent.adapterConfig), + projectPolicy: null, + issueSettings, + mode: resolvedMode, + legacyUseProjectWorkspace: null, + }, + ); + const resolvedStrategy = resolveEffectiveWorkspaceStrategyType( + resolvedMode, + workspaceManagedConfig, + ); + const existingExecutionWorkspaceStatus = issue.executionWorkspaceId + ? await tx + .select({ status: executionWorkspaces.status }) + .from(executionWorkspaces) + .where( + and( + eq(executionWorkspaces.id, issue.executionWorkspaceId), + eq(executionWorkspaces.companyId, issue.companyId), + ), + ) + .then((rows) => rows[0]?.status ?? null) + : null; + const reuseRequest = resolveExecutionWorkspaceReuseRequestForIssue({ + issueExecutionWorkspaceId: issue.executionWorkspaceId, + issueExecutionWorkspacePreference: + issue.executionWorkspacePreference, + existingExecutionWorkspaceStatus, + }); + const hasResolvablePriorSessionWorkspace = + await resolveHasResolvablePriorSessionWorkspace(); + + if ( + isUnrunnableWorktreeCombo({ + issue: { + projectId: issue.projectId ?? projectId ?? null, + projectWorkspaceId: issue.projectWorkspaceId, + executionWorkspaceId: issue.executionWorkspaceId, + executionWorkspacePreference: + issue.executionWorkspacePreference, + }, + resolvedMode, + resolvedStrategy, + reusableExecutionWorkspaceAvailable: + reuseRequest.existingExecutionWorkspaceAvailable, + hasResolvablePriorSessionWorkspace, + }) + ) { + if (caseRequest) waitForCaseAdmission(WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE); + const now = new Date(); + const issueLabel = formatIssueIdentifierLink( + issue.identifier, + issue.id, + ); + const blockedComment = [ + `Paperclip blocked ${issueLabel} before dispatch because its workspace settings are not runnable.`, + "", + `- Code: \`${WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE}\``, + `- Reason: ${WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE}`, + `- Next action: ${WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION}`, + ].join("\n"); + await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId: issue.id }); + await notifyNativeStatusReassessment(tx, { companyId: issue.companyId, issueId: issue.id }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + if (issue.status !== "blocked") await notifyChatResponseAvailability(tx, { companyId: issue.companyId, issueId: issue.id }); + await tx + .update(issues) + .set({ + status: "blocked", + checkoutRunId: null, + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: now, + }) + .where(eq(issues.id, issue.id)); + await insertIssueComment(tx, { + companyId: issue.companyId, + issueId: issue.id, + body: blockedComment, + createdAt: now, + updatedAt: now, + }); + await tx.insert(agentWakeupRequests).values({ + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, + payload: { + ...(payload ?? {}), + issueId, + heartbeatSkip: { + code: WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, + reason: WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE, + remediation: + WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION, + }, + }, + status: "skipped", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: now, + }); + await logActivity(tx as unknown as Db, { + companyId: issue.companyId, + actorType: "system", + actorId: "system", + agentId, + runId: null, + action: "issue.workspace_preflight_blocked", + entityType: "issue", + entityId: issue.id, + details: { + code: WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, + reason: WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE, + remediation: WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION, + requestedReason: reason, + source, + triggerDetail, + resolvedMode, + resolvedStrategy, + hasResolvablePriorSessionWorkspace, + }, + }); + return { kind: "skipped" as const }; + } + } + + if (activeExecutionRun) { + // The case ledger owns retry. Do not place its exact receipt in a + // generic deferred queue that cannot retain handoff authority. + if (caseRequest) return { kind: "deferred" as const }; + // The resolved action is already a durable retry outbox. Do not merge + // its fresh-session contract into unrelated work or create a second + // deferred wake that could later replay the same reconciliation. + if (reconciledSourceRunId) return { kind: "deferred" as const }; + + // Offer the input to the active external agent, but keep a durable queued wake + // until it reads the comment. A finish racing the event tick must + // leave unread input available for the next assignment. + const followUpProvider = parseObject(agent.adapterConfig).provider; + const museFollowUp = followUpProvider === "muse"; + const bindingId = readNonEmptyString(parseObject(agent.adapterConfig)[museFollowUp ? "museBindingId" : followUpProvider === "grok_bot" ? "externalBindingId" : "dotBindingId"]); + const followUpPayloadKey = museFollowUp ? "museAssignmentFollowUp" : "dotAssignmentFollowUp"; + let followUp: false | { assignmentId: string; consumed: boolean } = false; + let pendingFollowUps: typeof agentWakeupRequests.$inferSelect[] = []; + if (agent.adapterType === "paperclip_runner" && ["openai_dot", "grok_bot", "muse"].includes(String(followUpProvider)) + && bindingId && activeExecutionRun.agentId === agentId && issue.assigneeAgentId === agentId + && reason === "issue_commented" && wakeCommentId && opts.allowRunCoalescing !== false + && enrichedContextSnapshot.forceFreshSession !== true && !explicitResumeSession && !opts.manualUserWake + && !enrichedContextSnapshot.interactionId && !payload?.interactionId && !receiptRequest) { + // Receipt consumption uses the same issue -> wake -> run -> guard -> + // binding order. Lock before publishing, then reuse these wake rows. + pendingFollowUps = await tx.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId), + isNull(agentWakeupRequests.runId), eq(agentWakeupRequests.status, "deferred_issue_execution"), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`, + sql`${agentWakeupRequests.payload}->>${followUpPayloadKey} is not null`, + )).orderBy(asc(agentWakeupRequests.createdAt), asc(agentWakeupRequests.id)).for("update"); + const [currentRun] = await tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, activeExecutionRun.id), eq(heartbeatRuns.companyId, agent.companyId), + eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "running"), + eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.nativeIssueId, issue.id), + )).for("update"); + if (currentRun) followUp = await withExternalAdmissionGuard(tx, agent.companyId, agentId, + () => (museFollowUp ? publishActiveMuseComment : publishActiveDotComment)(tx as unknown as Db, { + companyId: agent.companyId, agentId, bindingId, runId: currentRun.id, + issueId: issue.id, commentId: wakeCommentId, + })); + } + if (followUp) { + const assignmentId = followUp.assignmentId; + const pending = followUp.consumed ? undefined : pendingFollowUps.find(wake => + wake.payload?.[followUpPayloadKey] === assignmentId && + wake.requestedByActorType === (opts.requestedByActorType ?? null) && + wake.requestedByActorId === (opts.requestedByActorId ?? null)); + if (pending) await tx.update(agentWakeupRequests).set({ + payload: withQueuedCommentIdsInWakePayload(pending.payload, + [...new Set([...queuedCommentIdsFromWakePayload(pending.payload), wakeCommentId!])]), + coalescedCount: pending.coalescedCount + 1, updatedAt: new Date(), + }).where(and(eq(agentWakeupRequests.id, pending.id), eq(agentWakeupRequests.companyId, agent.companyId), + eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"))); + else await tx.insert(agentWakeupRequests).values({ companyId: agent.companyId, agentId, source, triggerDetail, + reason, payload: withQueuedCommentIdsInWakePayload({ ...payload, issueId: issue.id, + [followUpPayloadKey]: followUp.assignmentId, + [DEFERRED_WAKE_CONTEXT_KEY]: enrichedContextSnapshot }, [wakeCommentId!]), + status: followUp.consumed ? "coalesced" : "deferred_issue_execution", + runId: followUp.consumed ? activeExecutionRun.id : null, + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: followUp.consumed ? new Date() : null }); + return { kind: "coalesced" as const, run: activeExecutionRun }; + } + + const admissionScope = wakeQueue.createAdmissionTransactionScope( + agent.companyId, + tx as unknown as Db, + ); + const admission = await wakeQueue.admitWakeBehindIssueExecution( + admissionScope, + { + companyId: agent.companyId, + issueId: issue.id, + agentId, + agentNameKey, + issueExecutionAgentNameKey: issue.executionAgentNameKey, + activeExecutionRun: { + id: activeExecutionRun.id, + agentId: activeExecutionRun.agentId, + status: activeExecutionRun.status, + contextSnapshot: activeExecutionRun.contextSnapshot, + wakeupRequestId: activeExecutionRun.wakeupRequestId, + }, + allowRunCoalescing: isConversation(issue) ? false : opts.allowRunCoalescing, + durableReceipt: receiptRequest + ? { + id: receiptRequest.id, + requestedAt: receiptRequest.requestedAt, + } + : undefined, + reason, + liveRunExecutions, + wakeCommentId, + forceFreshSession: + enrichedContextSnapshot.forceFreshSession === true, + contextSnapshot: enrichedContextSnapshot, + source, + triggerDetail, + payload, + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + }, + ); + + if (admission.kind === "coalesced") { + return { + kind: "coalesced" as const, + run: admission.run as typeof heartbeatRuns.$inferSelect, + }; + } + if (admission.kind === "deferred") { + return { kind: "deferred" as const, admitted: true }; + } + // admission.kind === "proceed": no active run absorbed this wake, + // so fall through to the ordinary queue path below. + } + + // PAP-13775: no live run holds the lock, so this wake would start a + // fresh adapter session. If this agent's recent runs on this issue + // keep succeeding without any issue-visible progress and the wake + // carries no new information, hold it back for an escalating cooldown + // so external pollers/reconcilers can't storm full-price sessions. + // Server-side recovery retries insert runs directly and never reach + // this gate. A verified durable case receipt carries a new phase; + // its agent-authored instruction comment is not an event-free wake. + if ( + !caseRequest && + isThrottleCandidateIssueRewake({ + reason, + wakeCommentId: wakeCommentId ?? null, + requestedByActorType: opts.requestedByActorType ?? null, + forceFreshSession: + enrichedContextSnapshot.forceFreshSession === true, + hasExplicitResume: Boolean(explicitResumeSession), + }) + ) { + const throttleNow = new Date(); + const recentTerminalRuns = await tx + .select({ + id: heartbeatRuns.id, + status: heartbeatRuns.status, + finishedAt: heartbeatRuns.finishedAt, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, agent.companyId), + eq(heartbeatRuns.agentId, agentId), + sql`${heartbeatRuns.finishedAt} is not null`, + gte( + heartbeatRuns.finishedAt, + new Date(throttleNow.getTime() - ISSUE_REWAKE_LOOKBACK_MS), + ), + sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`, + ), + ) + .orderBy(desc(heartbeatRuns.finishedAt)) + .limit(ISSUE_REWAKE_RUN_SAMPLE_LIMIT); + + if (recentTerminalRuns.length > 0) { + const sampleRunIds = recentTerminalRuns.map( + (sampleRun) => sampleRun.id, + ); + const progressRows = await tx + .select({ runId: activityLog.runId }) + .from(activityLog) + .where( + and( + eq(activityLog.companyId, agent.companyId), + eq(activityLog.entityType, "issue"), + eq(activityLog.entityId, issue.id), + inArray(activityLog.runId, sampleRunIds), + inArray( + activityLog.action, + ISSUE_PROGRESS_ACTIVITY_ACTIONS, + ), + ), + ); + const lastRunFinishedAt = + recentTerminalRuns[0]?.finishedAt ?? null; + const newInputRows = lastRunFinishedAt + ? await tx + .select({ id: activityLog.id }) + .from(activityLog) + .where( + and( + eq(activityLog.companyId, agent.companyId), + eq(activityLog.entityType, "issue"), + eq(activityLog.entityId, issue.id), + gt(activityLog.createdAt, lastRunFinishedAt), + sql`coalesce(${activityLog.details}->>'origin', '') <> 'fast_response'`, + inArray( + activityLog.action, + ISSUE_NEW_INPUT_ACTIVITY_ACTIONS, + ), + wakeCommentId && opts.requestedByActorType === "agent" + ? ne(activityLog.actorType, "agent") + : undefined, + ), + ) + .limit(1) + : []; + + const throttleDecision = evaluateIssueRewakeThrottle({ + now: throttleNow, + recentTerminalRuns, + runIdsWithIssueProgress: new Set( + progressRows + .map((row) => row.runId) + .filter((runId): runId is string => Boolean(runId)), + ), + // For an agent comment wake, the query excludes agent-authored + // activity while preserving genuinely new user/system input. + // Presentation/author metadata therefore cannot smuggle human + // wake privilege, nor can it mask an actual human response. + hasNewIssueInputSinceLastRun: newInputRows.length > 0, + }); + + if (throttleDecision.blocked) { + await tx.insert(agentWakeupRequests).values({ + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: "issue_rewake_throttled", + payload: { + ...(payload ?? {}), + issueId, + heartbeatSkip: { + reason: "issue_rewake_throttled", + requestedReason: reason, + noProgressStreak: throttleDecision.noProgressStreak, + cooldownMs: throttleDecision.cooldownMs, + lastRunFinishedAt: + throttleDecision.lastRunFinishedAt.toISOString(), + nextAllowedAt: + throttleDecision.nextAllowedAt.toISOString(), + }, + }, + status: "skipped", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: throttleNow, + }); + return { kind: "skipped" as const }; + } + } + } + + const dailyCapBlock = await getHeartbeatDailyCapBlock( + agent, + policy, + {}, + tx, + ); + if (dailyCapBlock) { + if (caseRequest) waitForCaseAdmission(dailyCapBlock.reason, "Case handoff is waiting for the agent's daily limit to clear"); + if (executionWaitRequestId && executionBlocker) { + continuationWait = { reason: dailyCapBlock.reason, + message: "The agent has reached its daily limit. Your message is saved until work can resume." }; + return deferBlockedExecution(executionBlocker); + } + const now = new Date(); + await tx.insert(agentWakeupRequests).values({ + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: dailyCapBlock.reason, + payload: { + ...(payload ?? {}), + heartbeatSkip: { + reason: + "Per-agent heartbeat daily cap reached before adapter invocation.", + observed: dailyCapBlock.observed, + limit: dailyCapBlock.limit, + }, + }, + status: "skipped", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: now, + }); + if (source === "timer") { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await tx + .update(agents) + .set({ + lastHeartbeatAt: now, + updatedAt: now, + }) + .where(eq(agents.id, agentId)); + } + return { kind: "skipped" as const }; + } + + const explicitContinuation = await admitExplicitNativeContinuation({ + db: tx as unknown as Db, companyId: issue.companyId, issueId: issue.id, + agentId, actorType: taskRetryActorId ? "user" : opts.requestedByActorType, actorId: taskRetryActorId ?? opts.requestedByActorId, + reason: taskRetryActorId ? "task_retry" : durableRequest && !failedChatRetry ? "issue_commented" : reason, + commentId: wakeCommentId ?? null, failedRunId: opts.failedRunId, successorRunId: explicitContinuationRunId, + queuedCommentInterruptId: opts.queuedCommentInterruptId, + queuedCommentRequestId: opts.queuedCommentRequestId, + taskRetryRequestId: opts.taskRetryRequestId, + ...(durableRequest?.resumeDeferred && !failedChatRetry ? { durableChatRequest: durableRequest } : {}), + onBlocked: (reason, message) => { continuationBlocked = true; continuationWait = { reason, message }; }, + }); + if (!explicitContinuation && (executionBlocker || (continuationBlocked && !verifiedConversationReset)) && !ownsReconstructionContinuation) + return deferBlockedExecution(continuationHold()); + if (explicitContinuation) { + enrichedContextSnapshot.forceFreshSession = true; + enrichedContextSnapshot.previousRunId = explicitContinuation.previousRunId; + enrichedContextSnapshot.explicitUserContinuation = explicitContinuation; + } + + const wakeupValues = { + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason, + payload, + status: "queued" as const, + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + }; + // A deferred durable chat wake already owns this ID and source. + // Promote that receipt in the same transaction as its successor. + const wakeupRequest = durableRequest && executionWaitRequestId === durableRequest.id + ? await tx.update(agentWakeupRequests).set({ + payload, status: "queued", error: null, finishedAt: null, updatedAt: new Date(), + }).where(and(eq(agentWakeupRequests.id, durableRequest.id), + eq(agentWakeupRequests.companyId, agent.companyId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), isNull(agentWakeupRequests.runId))) + .returning().then(rows => rows[0]) + : await tx.insert(agentWakeupRequests).values(wakeupValues).returning().then(rows => rows[0]); + if (!wakeupRequest) return { kind: "deferred" as const }; + + // A handoff changes the executor, not the owner of saved user input. + // Validate its exact stopped source while the issue row is locked; + // unrelated agents and dedicated continuations keep their own wakes. + const interruptedRunId = readNonEmptyString(enrichedContextSnapshot.interruptedRunId); + const handoffSource = source === "assignment" && reason === "issue_assigned" && + issue.assigneeAgentId === agentId && interruptedRunId && isUuidLike(interruptedRunId) + ? await tx.select({ agentId: heartbeatRuns.agentId }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, interruptedRunId), eq(heartbeatRuns.companyId, issue.companyId), + eq(heartbeatRuns.status, "cancelled"), eq(heartbeatRuns.errorCode, "issue_reassigned"), + ne(heartbeatRuns.agentId, agentId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}`, + or(isNull(heartbeatRuns.nativeIssueId), eq(heartbeatRuns.nativeIssueId, issue.id)), + )).then(rows => rows[0] ?? null) + : null; + const canCoalesceComments = !isConversation(issue) && opts.allowRunCoalescing !== false; + // A resumed receipt already passed admission under this issue lock. + // Consume it with its successor even when chat keeps other messages + // in separate turns, or finalization will deliver it a second time. + const pendingComments = + (executionWaitRequestId || canCoalesceComments) && + (ownsReconstructionContinuation || !(await getExecutionBlocker(tx as unknown as Db, issue.companyId, issue.id))) + ? await tx + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, issue.companyId), + inArray(agentWakeupRequests.agentId, handoffSource ? [agentId, handoffSource.agentId] : [agentId]), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`, + // Chat Retry already pins its complete authorized source + // batch. Keep separate board input for its own admission. + canCoalesceComments || (opts.taskRetryRequestId && !failedChatRetry) + ? undefined : eq(agentWakeupRequests.id, executionWaitRequestId!), + ), + ) + .orderBy(asc(agentWakeupRequests.requestedAt)) + : []; + const adoptedComments = pendingComments.filter((wake) => { + if (wake.id === executionWaitRequestId) return true; + if (wake.id === opts.queuedCommentInterruptId || wake.id === opts.queuedCommentRequestId) return true; + const deferredPayload = parseObject(wake.payload); + const deferredContext = parseObject( + deferredPayload[DEFERRED_WAKE_CONTEXT_KEY], + ); + // Dedicated interaction wakes carry their own source and session + // contract. ID-only adoption must not erase that continuation. + return ( + // Durable chat work must keep its receipt, actor, source, and + // session contract through normal promotion and authorization. + !wake.idempotencyKey?.startsWith("chat-inbound:") && + !isInteractionResolutionWakePayload(deferredPayload) && + !hasInteractionContinuationWakeContext(deferredContext) && + ["issue_commented", "issue_reopened_via_comment"].includes(String(deferredContext.wakeReason ?? wake.reason)) && + queuedCommentIdsFromWakePayload(wake.payload).length > 0 + ); + }); + let adoptedCommentIds = [ + ...new Set([ + ...adoptedComments.flatMap((wake) => + queuedCommentIdsFromWakePayload(wake.payload), + ), + ...queuedCommentIdsFromRunContext(enrichedContextSnapshot), + ]), + ]; + if (opts.queuedCommentRequestId || (opts.taskRetryRequestId && !failedChatRetry) || handoffSource) { + adoptedCommentIds = await undeliveredLegacyUserCommentIds(tx as unknown as Db, + agent.companyId, issueId, agentId, adoptedCommentIds); + } + await notifyChatRunMilestones(tx, { companyId: agent.companyId, issueId: issueId ?? undefined }); + const newRun = await tx + .insert(heartbeatRuns) + .values({ + ...(explicitContinuation ? { id: explicitContinuationRunId } : {}), + companyId: agent.companyId, + agentId, + scopeKind: readNonEmptyString(enrichedContextSnapshot.issueId) ? "issue" : "company", + issueId: readNonEmptyString(enrichedContextSnapshot.issueId), + invocationSource: source, + triggerDetail, + status: "queued", + responsibleUserId: await resolveQueuedResponsibleUserId(), + wakeupRequestId: wakeupRequest.id, + retryOfRunId: failedChatRetry + ? durableRequest!.failedRunRetry!.failedRunId + : opts.failedRunId ?? automaticParentRunId, + contextSnapshot: adoptedComments.length + ? withQueuedCommentIdsInRunContext( + enrichedContextSnapshot, + adoptedCommentIds, + adoptedComments.map((wake) => wake.id), + ) + : enrichedContextSnapshot, + sessionIdBefore: explicitContinuation ? null : sessionBefore, + continuationAttempt, + ...(reconciledSourceRunId + ? { retryOfRunId: reconciledSourceRunId } + : {}), + ...(reconciledRestoreRetryCount !== null ? { + scheduledRetryAttempt: reconciledRestoreRetryCount, + scheduledRetryReason: "transient_failure", + } : {}), + }) + .returning() + .then((rows) => rows[0]); + + if (ownsReconstructionContinuation) { + await tx.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ + continuationDelivery: "delivered", continuationRunId: newRun.id, + })}::jsonb` }).where(and(eq(issueRecoveryActions.companyId, issue.companyId), + eq(issueRecoveryActions.id, String(enrichedContextSnapshot.recoveryActionId)), + sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`)); + } + + await tx + .update(agentWakeupRequests) + .set({ + runId: newRun.id, + updatedAt: new Date(), + }) + .where(eq(agentWakeupRequests.id, wakeupRequest.id)); + + if (failedChatRetry && opts.taskRetryRequestId) { + await tx.update(agentWakeupRequests).set({ + status: "coalesced", runId: newRun.id, finishedAt: new Date(), updatedAt: new Date(), + }).where(and(eq(agentWakeupRequests.companyId, issue.companyId), + eq(agentWakeupRequests.id, opts.taskRetryRequestId), + eq(agentWakeupRequests.status, "deferred_issue_execution"))); + } + + if (adoptedComments.length) { + await tx + .update(agentWakeupRequests) + .set({ + status: "coalesced", + runId: newRun.id, + finishedAt: new Date(), + updatedAt: new Date(), + }) + .where( + inArray( + agentWakeupRequests.id, + adoptedComments.map((wake) => wake.id), + ), + ); + await tx + .update(agentWakeupRequests) + .set({ + payload: withQueuedCommentIdsInWakePayload(payload, adoptedCommentIds), + }) + .where(eq(agentWakeupRequests.id, wakeupRequest.id)); + } + + // executionRunId is NOT stamped here (enqueueWakeup queues the run but + // doesn't start it). It will be stamped in claimQueuedRun() once the run + // transitions to "running" — Fix A (lazy locking). + + return { kind: "queued" as const, run: newRun }; + })) + .catch((error) => { + if (isExternalChatWaitAuthorizationContention(error)) + return { kind: "deferred" as const }; + throw error; + }); + + // Telemetry for the cancelled runs is best-effort background work. + // Fire it here and never await it: none of the lifecycle work below, + // nor this function's return, depends on it, so a slow telemetry + // lookup must not delay them. + for (const cancelledRun of cancelledRunsToEmit) { + void emitAgentTaskRun(db, cancelledRun); + } + + if (outcome.kind === "durable") { + return outcome.receipt.runId ? getRun(outcome.receipt.runId) : null; + } + if (outcome.kind === "deferred" || outcome.kind === "skipped") { + return null; + } + if (outcome.kind === "coalesced") { + await startNextQueuedRunForAgent(agent.id); + return outcome.run; + } + if (outcome.kind === "replayed") { + if (outcome.run.status === "queued") + await startNextQueuedRunForAgent(agent.id); + return outcome.run; + } + + const newRun = outcome.run; + publishLiveEvent({ + companyId: newRun.companyId, + type: "heartbeat.run.queued", + payload: { + runId: newRun.id, + agentId: newRun.agentId, + invocationSource: newRun.invocationSource, + triggerDetail: newRun.triggerDetail, + wakeupRequestId: newRun.wakeupRequestId, + }, + }); + + await startNextQueuedRunForAgent(agent.id); + return newRun; + } + + if (durableRequest) throw new Error("chat_inbound_wakeup_binding_denied"); + + const activeRuns = await db + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.agentId, agentId), + inArray(heartbeatRuns.status, [ + ...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES, + ]), + ), + ) + .orderBy(desc(heartbeatRuns.createdAt)); + + const sameScopeQueuedRun = activeRuns.find( + (candidate) => + candidate.status === "queued" && + isSameTaskScope(runTaskKey(candidate), taskKey), + ); + const sameScopeScheduledRetryRun = activeRuns.find( + (candidate) => + candidate.status === "scheduled_retry" && + isSameTaskScope(runTaskKey(candidate), taskKey), + ); + const sameScopeRunningRun = activeRuns.find( + (candidate) => + candidate.status === "running" && + isSameTaskScope(runTaskKey(candidate), taskKey), + ); + const shouldQueueFollowupForRunningWake = + Boolean(sameScopeRunningRun) && + !sameScopeQueuedRun && + shouldQueueFollowupForRunningIssueWake({ + contextSnapshot: enrichedContextSnapshot, + wakeCommentId, + }); + // Unscoped manual wakes need their own receipt and execution identity too. + const rawCoalescedTarget = + opts.allowRunCoalescing === false || opts.manualUserWake + ? null + : (sameScopeQueuedRun ?? + sameScopeScheduledRetryRun ?? + (shouldQueueFollowupForRunningWake + ? null + : (sameScopeRunningRun ?? null))); + + // A reviewed chat run attests one conversation, so never fold another + // conversation's reply into it; that reply queues its own run instead. + const coalescedTargetRun = rawCoalescedTarget && + wakeChatConversationsConflict(rawCoalescedTarget.contextSnapshot, enrichedContextSnapshot) + ? null + : filterZombieCoalesceTarget(rawCoalescedTarget, liveRunExecutions); + + if (coalescedTargetRun) { + const mergedContextSnapshot = mergeCoalescedContextSnapshot( + coalescedTargetRun.contextSnapshot, + enrichedContextSnapshot, + { + preserveExistingInteractionContinuation: + coalescedTargetRun.status === "queued" || + coalescedTargetRun.status === "scheduled_retry", + }, + ); + const mergedRun = await db + .update(heartbeatRuns) + .set({ + contextSnapshot: mergedContextSnapshot, + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, coalescedTargetRun.id)) + .returning() + .then((rows) => rows[0] ?? coalescedTargetRun); + + await db.insert(agentWakeupRequests).values({ + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason, + payload, + status: "coalesced", + coalescedCount: 1, + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + runId: mergedRun.id, + finishedAt: new Date(), + }); + return mergedRun; + } + + const queueOutcome = await db.transaction(async (tx) => { + await notifyRunDispatch(tx, { companyId: agent.companyId }); + await tx.execute( + sql`select id from agents where id = ${agentId} and company_id = ${agent.companyId} for update`, + ); + + const dailyCapBlock = await getHeartbeatDailyCapBlock( + agent, + policy, + {}, + tx, + ); + if (dailyCapBlock) { + const now = new Date(); + await tx.insert(agentWakeupRequests).values({ + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: dailyCapBlock.reason, + payload: { + ...(payload ?? {}), + heartbeatSkip: { + reason: + "Per-agent heartbeat daily cap reached before adapter invocation.", + observed: dailyCapBlock.observed, + limit: dailyCapBlock.limit, + }, + }, + status: "skipped", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: now, + }); + if (source === "timer") { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await tx + .update(agents) + .set({ + lastHeartbeatAt: now, + updatedAt: now, + }) + .where(eq(agents.id, agentId)); + } + return { kind: "skipped" as const }; + } + + const wakeupRequest = await tx + .insert(agentWakeupRequests) + .values({ + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason, + payload, + status: "queued", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + }) + .returning() + .then((rows) => rows[0]); + + const newRun = await tx + .insert(heartbeatRuns) + .values({ + companyId: agent.companyId, + agentId, + scopeKind: readNonEmptyString(enrichedContextSnapshot.issueId) ? "issue" : "company", + issueId: readNonEmptyString(enrichedContextSnapshot.issueId), + invocationSource: source, + triggerDetail, + status: "queued", + responsibleUserId: await resolveQueuedResponsibleUserId(), + wakeupRequestId: wakeupRequest.id, + contextSnapshot: enrichedContextSnapshot, + sessionIdBefore: sessionBefore, + continuationAttempt, + }) + .returning() + .then((rows) => rows[0]); + + await tx + .update(agentWakeupRequests) + .set({ + runId: newRun.id, + updatedAt: new Date(), + }) + .where(eq(agentWakeupRequests.id, wakeupRequest.id)); + + return { kind: "queued" as const, run: newRun }; + }); + + if (queueOutcome.kind === "skipped") return null; + const newRun = queueOutcome.run; + + publishLiveEvent({ + companyId: newRun.companyId, + type: "heartbeat.run.queued", + payload: { + runId: newRun.id, + agentId: newRun.agentId, + invocationSource: newRun.invocationSource, + triggerDetail: newRun.triggerDetail, + wakeupRequestId: newRun.wakeupRequestId, + }, + }); + + await startNextQueuedRunForAgent(agent.id); + + return newRun; + } + + /** + * Native status commitment deliberately persists dependency/parent wake + * intents in the same transaction as the authoritative status projection. + * Those rows are not runnable until the heartbeat scheduler has applied its + * normal policy, workspace, concurrency, and responsible-user checks. Bridge + * the durable intent into that scheduler here instead of treating a bare + * `agent_wakeup_requests` row as if it were already a queued heartbeat run. + * + * The intent is claimed before dispatch. A deterministic dispatcher actor id + * lets a later sweep recover the narrow process-crash window after the real + * wake was inserted but before the intent was linked to it. Dispatch failure + * only requeues the intent; it never changes the provider run outcome. + */ + async function dispatchPendingNativeStatusWakeups( + input: { + companyId?: string; + limit?: number; + staleClaimMs?: number; + } = {}, + ) { + const now = new Date(); + const staleClaimMs = Math.max(1_000, input.staleClaimMs ?? 60_000); + const candidates = await db + .select() + .from(agentWakeupRequests) + .where( + and( + input.companyId + ? eq(agentWakeupRequests.companyId, input.companyId) + : undefined, + eq(agentWakeupRequests.requestedByActorType, "system"), + eq(agentWakeupRequests.requestedByActorId, "native-status-committer"), + inArray(agentWakeupRequests.status, ["queued", "claimed"]), + isNull(agentWakeupRequests.runId), + ), + ) + .orderBy(asc(agentWakeupRequests.requestedAt)) + .limit(Math.max(1, Math.min(input.limit ?? 100, 500))); + + let dispatched = 0; + let recovered = 0; + let deferred = 0; + const deliveredByIssueScope = new Map< + string, + { runId: string | null; status: string } + >(); + + for (const candidate of candidates) { + const payload = parseObject(candidate.payload); + const wakeContext = parseObject(payload._paperclipWakeContext); + const issueId = + readNonEmptyString(payload.issueId) ?? + readNonEmptyString(payload.taskId) ?? + readNonEmptyString(wakeContext.issueId) ?? + null; + const dispatchActorId = `native-status-wake-dispatch:${candidate.id}`; + const existingDispatch = await db + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, candidate.companyId), + eq(agentWakeupRequests.requestedByActorType, "system"), + eq(agentWakeupRequests.requestedByActorId, dispatchActorId), + ), + ) + .orderBy(desc(agentWakeupRequests.requestedAt)) + .limit(1) + .then((rows) => rows[0] ?? null); + + if (existingDispatch) { + const recoveredStatus = existingDispatch.runId + ? "coalesced" + : existingDispatch.status === "deferred_issue_execution" + ? "coalesced" + : existingDispatch.status; + await db.transaction(async tx => { + await notifyChatRunMilestones(tx, { companyId: candidate.companyId, issueId: issueId ?? undefined }); + await tx + .update(agentWakeupRequests) + .set({ + status: recoveredStatus, + runId: existingDispatch.runId, + finishedAt: + (existingDispatch.runId ?? existingDispatch.finishedAt) + ? (existingDispatch.finishedAt ?? now) + : null, + error: existingDispatch.error, + updatedAt: now, + }) + .where( + and( + eq(agentWakeupRequests.id, candidate.id), + isNull(agentWakeupRequests.runId), + ), + ); + }); + recovered += 1; + continue; + } + + if ( + candidate.status === "claimed" && + candidate.claimedAt && + now.getTime() - candidate.claimedAt.getTime() < staleClaimMs + ) { + deferred += 1; + continue; + } + + const claimed = await db + .update(agentWakeupRequests) + .set({ + status: "claimed", + claimedAt: now, + error: null, + updatedAt: now, + }) + .where( + and( + eq(agentWakeupRequests.id, candidate.id), + isNull(agentWakeupRequests.runId), + candidate.status === "claimed" + ? eq(agentWakeupRequests.status, "claimed") + : eq(agentWakeupRequests.status, "queued"), + ), + ) + .returning({ id: agentWakeupRequests.id }); + if (claimed.length === 0) continue; + + const scopeKey = issueId + ? `${candidate.companyId}:${candidate.agentId}:${issueId}:${readNativeReviewAssignmentContext(wakeContext)?.nativeReviewInteractionId ?? ""}` + : null; + const priorDelivery = scopeKey + ? deliveredByIssueScope.get(scopeKey) + : null; + if (priorDelivery) { + await db.transaction(async tx => { + await notifyChatRunMilestones(tx, { companyId: candidate.companyId, issueId: issueId ?? undefined }); + await tx + .update(agentWakeupRequests) + .set({ + status: "coalesced", + runId: priorDelivery.runId, + finishedAt: new Date(), + error: null, + updatedAt: new Date(), + }) + .where(eq(agentWakeupRequests.id, candidate.id)); + }); + recovered += 1; + continue; + } + + let completedOnboardingGuard: WakeupOptions["issueStateGuard"]; + if (issueId) { + const targetIssue = await db + .select({ + status: issues.status, + statusVersion: issues.statusVersion, + assigneeAgentId: issues.assigneeAgentId, + }) + .from(issues) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, candidate.companyId), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + const nativeReview = candidate.reason === "native_completion_review" + ? await getNativeReviewAssignment(db, { + companyId: candidate.companyId, issueId, agentId: candidate.agentId, + contextSnapshot: wakeContext, + }) + : null; + const onboardingResultReport = targetIssue?.status === "done" && await isCompletedOnboardingHandoffWake(db, { + companyId: candidate.companyId, issueId, agentId: candidate.agentId, + reason: candidate.reason, contextSnapshot: wakeContext, + }); + if (onboardingResultReport) completedOnboardingGuard = { assigneeAgentId: candidate.agentId, statuses: ["done"], statusVersion: targetIssue!.statusVersion }; + if ( + !targetIssue || + (["done", "cancelled"].includes(targetIssue.status) && !onboardingResultReport) || + (targetIssue.assigneeAgentId !== candidate.agentId && !nativeReview) || + (candidate.reason === "native_completion_review" && !nativeReview) + ) { + await db.transaction(async tx => { + await notifyChatInboundWakeNotices(tx, { companyId: candidate.companyId, issueId }); + await tx + .update(agentWakeupRequests) + .set({ + status: "skipped", + finishedAt: new Date(), + error: !targetIssue + ? "Native status wake target no longer exists" + : ["done", "cancelled"].includes(targetIssue.status) + ? `Native status wake target is already ${targetIssue.status}` + : "Native status wake target has a different assignee", + updatedAt: new Date(), + }) + .where(eq(agentWakeupRequests.id, candidate.id)); + }); + recovered += 1; + continue; + } + } + + try { + const wakeRun = await enqueueWakeup(candidate.agentId, { + allowRunCoalescing: false, + source: candidate.source as WakeupOptions["source"], + triggerDetail: (candidate.triggerDetail ?? + "system") as WakeupOptions["triggerDetail"], + reason: candidate.reason, + payload, + idempotencyKey: candidate.idempotencyKey, + requestedByActorType: "system", + requestedByActorId: dispatchActorId, + ...(completedOnboardingGuard ? { issueStateGuard: completedOnboardingGuard } : {}), + contextSnapshot: { + ...wakeContext, + ...(issueId ? { issueId, taskId: issueId } : {}), + wakeReason: candidate.reason, + source: "native_status_decision", + statusDecisionSource: "native_status_decision", + nativeStatusWakeIntentId: candidate.id, + }, + }); + + const delivered = await db + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, candidate.companyId), + eq(agentWakeupRequests.requestedByActorType, "system"), + eq(agentWakeupRequests.requestedByActorId, dispatchActorId), + ), + ) + .orderBy(desc(agentWakeupRequests.requestedAt)) + .limit(1) + .then((rows) => rows[0] ?? null); + + // The committer intent is the durable outbox entry. When admission is + // blocked, enqueueWakeup creates a separate deferred dispatch receipt; + // leave the original intent coalesced so the release drain cannot + // promote both rows (the original has no nativeStatusWakeIntentId + // provenance and would otherwise run once before the dispatch receipt). + const deliveredStatus = delivered?.status ?? "queued"; + await db.transaction(async tx => { + await notifyChatRunMilestones(tx, { companyId: candidate.companyId, issueId: issueId ?? undefined }); + await tx + .update(agentWakeupRequests) + .set({ + status: wakeRun || deliveredStatus === "deferred_issue_execution" + ? "coalesced" + : deliveredStatus, + runId: wakeRun?.id ?? delivered?.runId ?? null, + finishedAt: + wakeRun || delivered?.finishedAt + ? (delivered?.finishedAt ?? new Date()) + : null, + error: delivered?.error ?? null, + updatedAt: new Date(), + }) + .where(eq(agentWakeupRequests.id, candidate.id)); + }); + + if (scopeKey) { + deliveredByIssueScope.set(scopeKey, { + runId: wakeRun?.id ?? delivered?.runId ?? null, + status: wakeRun ? "coalesced" : (delivered?.status ?? "queued"), + }); + } + + if (wakeRun) dispatched += 1; + else deferred += 1; + } catch (error) { + await db + .update(agentWakeupRequests) + .set({ + status: "queued", + claimedAt: null, + error: + error instanceof Error + ? error.message.slice(0, 1_000) + : String(error).slice(0, 1_000), + updatedAt: new Date(), + }) + .where( + and( + eq(agentWakeupRequests.id, candidate.id), + eq(agentWakeupRequests.status, "claimed"), + isNull(agentWakeupRequests.runId), + ), + ); + logger.warn( + { + err: error, + wakeupRequestId: candidate.id, + agentId: candidate.agentId, + }, + "failed to dispatch persisted native status wake intent", + ); + deferred += 1; + } + } + + return { scanned: candidates.length, dispatched, recovered, deferred }; + } + + return { + enqueueWakeup, + dispatchPendingNativeStatusWakeups, + startNextQueuedRunForAgent, + countRunningRunsForAgent, + releaseRunClaimedJustBeforeSuppression, + claimQueuedRun, + withChatControlRecoveryGate, + trackWakeup, + resumeQueuedRuns, + resumeQueuedRunsAfterDrain, + parseHeartbeatPolicy, + nextTimerHeartbeatAt, + claimDueTimerHeartbeat, + }; +} diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 0e6a67a4dcb..d2e72ab0b6e 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -1 +1,1934 @@ +import { notifyRunDispatch } from "../run-dispatch-notifications.js"; import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; +import { deriveTaskKeyWithHeartbeatFallback } from "./run-state.js"; +import { infrastructureRetryDelayMs } from "../system-recovery-state.js"; +import { + WORKSPACE_VALIDATION_FAILURE_CODE, + isWorkspaceValidationFailedRun, + readWorkspaceValidationPayloadFromRun, +} from "./workspaces.js"; +import { admitExplicitContinuationRetry } from "../explicit-native-continuation.js"; +import { + CONVERSATION_CONTINUATION_POLICY, + hasConversationContinuationPolicy, +} from "../conversation-continuation.js"; +import { legacyExecutionNeedsReconciliationWithEvidence } from "../legacy-execution-recovery.js"; +import { + executionFailureRetryCount, + executionRetryAttemptCount, + accountingForScheduledRetry, +} from "../execution-recovery-attempt.js"; +import { WORKSPACE_GIT_SCAN_ERROR_CODES } from "../workspace-git-operation-scheduler.js"; +import { CONTROLLER_LEASE_LOST_FAILURE_CODE } from "../legacy-controller-lease.js"; +import { randomUUID } from "node:crypto"; +import { + and, + asc, + desc, + eq, + inArray, + ne, + notInArray, + or, + sql, +} from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { + agents, + agentWakeupRequests, + executionWorkspaces, + heartbeatRuns, + issues, +} from "@paperclipai/db"; +import { + HttpError, + notFound, +} from "../../errors.js"; +import { logger } from "../../middleware/logger.js"; +import { parseObject } from "../../adapters/utils.js"; +import { logActivity } from "../activity-log.js"; +import { readContinuationAttempt } from "../recovery/index.js"; +import { withRecoveryContext } from "../recovery/status-only-context.js"; +import { + MAX_TURN_CONTINUATION_RETRY_REASON, + WORKSPACE_BUSY_RETRY_REASON, + AI_CONNECTION_BUSY_RETRY_REASON, + AI_CONNECTION_POOL_WAIT_RETRY_REASON, + INTERACTION_CONTINUATION_INFRA_RETRY_REASON, + INTERACTION_CONTINUATION_INFRA_WAKE_REASON, + isNonAssigneeWorkspaceBusyRetry, + isResolvedInteractionContinuationWakeContext, +} from "../../modules/run-dispatch/index.js"; + +import type { AppendHeartbeatRunEventInput } from "../heartbeat-run-events.js"; +import type { evaluateAgentInvokabilityFromDb } from "../agent-invokability.js"; +import type { createHeartbeatRunState } from "./run-state.js"; +import type { createHeartbeatRunPreparation } from "./run-preparation.js"; +import type { createRunDispatch, PostCommitEffect } from "../../modules/run-dispatch/index.js"; + +type HeartbeatRun = typeof heartbeatRuns.$inferSelect; +type HeartbeatRunState = ReturnType; +type PlanApprovalResumeFailure = { + run: HeartbeatRun; + issueId: string | null; + attempt: number; + maxAttempts: number; +}; + +/** The retry module owns scheduling writes; the service owns lifecycle effects. */ +export interface HeartbeatRetryDependencies { + hasCaseHandoffForRun: (run: Pick) => Promise; + runDispatch: Pick, + "evaluateScheduledRetryGate" | "promoteDueScheduledRetries" | "promoteScheduledRetry" | "nextScheduledRetryAt">; + getAgent: (agentId: string) => Promise; + getRun: ( + runId: string, + options?: Parameters[1], + ) => Promise> | null | undefined>; + resolveSessionBeforeForWakeup: HeartbeatRunState["resolveSessionBeforeForWakeup"]; + resolveResponsibleUserIdForRunContext: ReturnType["resolveResponsibleUserIdForRunContext"]; + getAgentInvokability: (agent: typeof agents.$inferSelect | null | undefined) => ReturnType; + appendRunEvent: (run: HeartbeatRun, event: { + eventType: string; + stream?: "system" | "stdout" | "stderr"; + level?: "info" | "warn" | "error"; + color?: string; + message?: string; + payload?: Record; + retryExhaustion?: AppendHeartbeatRunEventInput["retryExhaustion"]; + }) => Promise; + escalatePlanApprovalResumeFailureNeedsAttention: (input: PlanApprovalResumeFailure) => Promise; + recordPlanApprovalResumeFailureRetry: (input: PlanApprovalResumeFailure & { retryRunId: string | null }) => Promise; + setRunStatusIfRunning: (runId: string, status: string, patch?: Partial) => Promise<{ + updated: boolean; + run: HeartbeatRun | null; + }>; + setWakeupStatus: (wakeupRequestId: string | null | undefined, status: string, patch?: Partial) => Promise; + releaseIssueExecutionAndPromote: (run: Pick) => Promise; + finalizeAgentStatus: ( + agentId: string, + outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out", + failureReason?: string | null, + options?: { keepIdleOnFailure?: boolean; wasFirstHeartbeat?: boolean }, + ) => Promise; + getWorktreeExecutionCutoff: () => Promise; + applyRunDispatchPostCommitEffects: (effects: PostCommitEffect[]) => void; +} + +export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS = [ + 30_000, 60_000, +] as const; + +const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON = "transient_failure"; + +const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry"; + +export function isTransientWorkspaceGitScanCode(code: string | null | undefined): boolean { + return code === WORKSPACE_GIT_SCAN_ERROR_CODES.timeout || code === WORKSPACE_GIT_SCAN_ERROR_CODES.saturated; +} + +export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = + BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length; + +const INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS = 2; + +const MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES = [ + "scheduled_retry", + "queued", + "running", +] as const; + +export const WORKSPACE_BUSY_RETRY_WAKE_REASON = "workspace_busy_retry"; + +export const WORKSPACE_BUSY_ERROR_CODE = "workspace_busy"; + +export const WORKSPACE_BUSY_RETRY_BASE_DELAY_MS = 60 * 1000; + +export const WORKSPACE_BUSY_RETRY_JITTER_MS = 60 * 1000; + +// Preserve the one-hour shared-workspace holder cutoff independently of the +// informational output-silence warnings. Warning sooner must not let another +// run overtake a quiet holder and mutate its shared workspace. +export const WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS = 60 * 60 * 1000; + +// Issue-level executionWorkspaceSettings.mode values that unambiguously opt an +// issue's runs out of the shared project workspace, and therefore out of +// shared-workspace serialization ("isolated" is the legacy alias +// parseIssueExecutionWorkspaceSettings normalizes to isolated_workspace). Any +// other value — including agent_default and an absent mode — may still resolve +// to the shared workspace and counts as a holder. +const ISOLATED_EXECUTION_WORKSPACE_MODES = [ + "isolated_workspace", + "operator_branch", + "isolated", +] as const; + +type CodexTransientFallbackMode = + | "same_session" + | "safer_invocation" + | "fresh_session" + | "fresh_session_safer_invocation"; + +export interface SharedWorkspaceHolder { + runId: string; + agentId: string; + issueId: string; + issueIdentifier: string | null; +} + +export interface ExecutionWorkspaceHolder extends SharedWorkspaceHolder { + // Silent past WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS but still running. + stale: boolean; +} + +// Pre-dispatch gate outcome: another running run currently holds the issue's +// shared project workspace, or the shared execution workspace it reuses. Not a +// failure — the run is parked as a bounded scheduled retry and re-attempted +// once the holder finishes, so two agents never mutate the same working tree +// concurrently. +export class WorkspaceBusyDeferral extends Error { + code = WORKSPACE_BUSY_ERROR_CODE; + holder: SharedWorkspaceHolder; + projectWorkspaceId: string | null; + executionWorkspaceId: string | null; + deferralAttempt: number; + wasIssueAssignee: boolean; + + constructor(input: { + holder: SharedWorkspaceHolder; + projectWorkspaceId?: string | null; + executionWorkspaceId?: string | null; + deferralAttempt: number; + wasIssueAssignee: boolean; + }) { + super( + `Shared ${input.executionWorkspaceId ? "execution" : "project"} workspace is busy: run ${input.holder.runId} (issue ${ + input.holder.issueIdentifier ?? input.holder.issueId + }) is still running`, + ); + this.name = "WorkspaceBusyDeferral"; + this.holder = input.holder; + this.projectWorkspaceId = input.projectWorkspaceId ?? null; + this.executionWorkspaceId = input.executionWorkspaceId ?? null; + this.deferralAttempt = input.deferralAttempt; + this.wasIssueAssignee = input.wasIssueAssignee; + } +} + +export function isWorkspaceBusyDeferral( + error: unknown, +): error is WorkspaceBusyDeferral { + return error instanceof WorkspaceBusyDeferral; +} + +export function computeWorkspaceBusyRetryDelayMs( + random: () => number = Math.random, +) { + const jitter = Math.min(Math.max(random(), 0), 1); + return ( + WORKSPACE_BUSY_RETRY_BASE_DELAY_MS + + Math.floor(jitter * WORKSPACE_BUSY_RETRY_JITTER_MS) + ); +} + +function resolveCodexTransientFallbackMode( + attempt: number, +): CodexTransientFallbackMode { + if (attempt <= 1) return "same_session"; + if (attempt === 2) return "safer_invocation"; + if (attempt === 3) return "fresh_session"; + return "fresh_session_safer_invocation"; +} + +export function readHeartbeatRunErrorFamily( + run: Pick, +) { + const resultJson = parseObject(run.resultJson); + const persistedFamily = readNonEmptyString(resultJson.errorFamily); + if (persistedFamily) return persistedFamily; + + if (run.errorCode === "provider_quota") { + return "provider_quota"; + } + if ( + run.errorCode === "codex_transient_upstream" || + run.errorCode === "claude_transient_upstream" || + run.errorCode === "codex_harness_crash" + ) { + return "transient_upstream"; + } + return null; +} + +function readTransientRetryNotBeforeFromRun( + run: Pick, +) { + const resultJson = parseObject(run.resultJson); + const value = resultJson.retryNotBefore ?? resultJson.transientRetryNotBefore; + if (!( + typeof value === "string" || + typeof value === "number" || + value instanceof Date + )) { + return null; + } + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? null : parsed; +} + +export function readTransientRecoveryContractFromRun( + run: Pick, +) { + const errorFamily = readHeartbeatRunErrorFamily(run); + return errorFamily === "transient_upstream" || + errorFamily === "provider_quota" + ? { + errorFamily, + retryNotBefore: readTransientRetryNotBeforeFromRun(run), + } + : null; +} + +function isSpawnLikeFailureMessage(value: unknown) { + if (typeof value !== "string") return false; + return /failed to start command|spawn\b|\bENOENT\b/i.test(value); +} + +// A sandbox provider plugin's worker can be briefly down during its own +// restart window (e.g. a rolling deploy of the plugin worker process). Lease +// acquisition fails immediately in that window, but the condition is +// transient and self-healing, so it must be treated as retryable +// infrastructure rather than a terminal setup failure. See +// resolveSandboxProviderPlugin's "worker_unavailable" message in +// environment-runtime.ts (":808"), e.g. 'Sandbox provider "kubernetes" is +// installed via plugin "acme.kubernetes-sandbox-provider", but its worker is +// not running.' +// +// This is anchored on both "is installed via plugin" and "but its worker is +// not running" so it does not also match plugin-environment-driver.ts's +// unrelated, permanent "provider not installed" message ('Sandbox provider +// "X" is not installed or its plugin worker is not running.'), which +// coincidentally contains the same "worker is not running" substring but +// describes a terminal condition that must not be retried. +function isSandboxProviderWorkerUnavailableFailureMessage(value: unknown) { + if (typeof value !== "string") return false; + return /sandbox provider .* is installed via plugin .* but its worker is not running/i.test( + value, + ); +} + +function isRetryableInteractionContinuationInfrastructureFailure( + run: Pick< + typeof heartbeatRuns.$inferSelect, + "error" | "errorCode" | "resultJson" + >, +) { + // `controller_lease_lost` is only recorded before provider dispatch, so the + // retry cannot repeat started work. + if ( + run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE || + run.errorCode === "process_lost" || + run.errorCode === CONTROLLER_LEASE_LOST_FAILURE_CODE + ) { + return true; + } + + if (run.errorCode !== "adapter_failed" && run.errorCode !== "setup_failed") + return false; + + const resultJson = parseObject(run.resultJson); + return ( + isSpawnLikeFailureMessage(run.error) || + isSpawnLikeFailureMessage(resultJson.errorMessage) || + isSpawnLikeFailureMessage(resultJson.message) || + isSandboxProviderWorkerUnavailableFailureMessage(run.error) || + isSandboxProviderWorkerUnavailableFailureMessage(resultJson.errorMessage) || + isSandboxProviderWorkerUnavailableFailureMessage(resultJson.message) + ); +} + +export function computeBoundedTransientHeartbeatRetrySchedule( + attempt: number, + now = new Date(), + random: () => number = Math.random, +) { + if (!Number.isInteger(attempt) || attempt <= 0) return null; + const baseDelayMs = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS[attempt - 1]; + if (typeof baseDelayMs !== "number") return null; + const delayMs = infrastructureRetryDelayMs(attempt, random()); + return { + attempt, + baseDelayMs, + delayMs, + dueAt: new Date(now.getTime() + delayMs), + maxAttempts: BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS, + }; +} + +export function normalizeAgentNameKey(value: string | null | undefined) { + if (typeof value !== "string") return null; + const normalized = value.trim().toLowerCase(); + return normalized.length > 0 ? normalized : null; +} + +function readNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +/** Retry persistence uses the service database and explicit lifecycle callbacks. */ +export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDependencies) { + const { + hasCaseHandoffForRun, + appendRunEvent, + escalatePlanApprovalResumeFailureNeedsAttention, + getAgentInvokability, + runDispatch, + resolveSessionBeforeForWakeup, + resolveResponsibleUserIdForRunContext, + recordPlanApprovalResumeFailureRetry, + setRunStatusIfRunning, + setWakeupStatus, + getRun, + getAgent, + releaseIssueExecutionAndPromote, + finalizeAgentStatus, + getWorktreeExecutionCutoff, + applyRunDispatchPostCommitEffects, + } = dependencies; + + async function scheduleBoundedRetryForRun( + run: typeof heartbeatRuns.$inferSelect, + agent: typeof agents.$inferSelect, + opts?: { + now?: Date; + random?: () => number; + retryReason?: string; + wakeReason?: string; + maxAttempts?: number; + delayMs?: number; + }, + ) { + if (await hasCaseHandoffForRun(run)) return { outcome: "not_scheduled" as const, + reason: "The durable case handoff owns successor work.", issueId: readNonEmptyString(run.contextSnapshot?.issueId) }; + if (["openai_dot", "muse"].includes(String(parseObject(agent.adapterConfig).provider)) + || ["openai_dot", "muse"].includes(String(parseObject(parseObject(parseObject(run.runnerProfileJson).nativeExecutionInput).provider).kind))) { + return { outcome: "not_scheduled" as const, reason: "Dot external execution must be reconciled before a new assignment; Paperclip cannot confirm its external stop.", issueId: readNonEmptyString(run.contextSnapshot?.issueId) }; + } + if (run.errorCode === "provider_tool_definition_invalid") { + return { outcome: "not_scheduled" as const, + reason: "Repair the invalid tool definitions before starting a new attempt.", + issueId: readNonEmptyString(run.contextSnapshot?.issueId) }; + } + if (Array.isArray(run.contextSnapshot?.chatCompletionDeliveryIds) && + run.contextSnapshot.chatCompletionDeliveryIds.some(id => typeof id === "string")) { + return { outcome: "not_scheduled" as const, reason: "The completion outbox owns this reply's retry budget and publication identity.", + errorCode: "chat_completion_outbox_owns_retry" as const, issueId: readNonEmptyString(run.contextSnapshot.issueId) }; + } + const now = opts?.now ?? new Date(); + const retryReason = + opts?.retryReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON; + const wakeReason = + opts?.wakeReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON; + const maxAttempts = Math.max( + 0, + Math.floor( + opts?.maxAttempts ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS, + ), + ); + const consumedAttempts = executionRetryAttemptCount(run, retryReason); + const nextAttempt = consumedAttempts + 1; + const computedBaseSchedule = + opts?.delayMs != null + ? nextAttempt <= maxAttempts + ? { + attempt: nextAttempt, + baseDelayMs: Math.max(0, Math.floor(opts.delayMs)), + delayMs: Math.max(0, Math.floor(opts.delayMs)), + dueAt: new Date( + now.getTime() + Math.max(0, Math.floor(opts.delayMs)), + ), + maxAttempts, + } + : null + : nextAttempt <= maxAttempts + ? computeBoundedTransientHeartbeatRetrySchedule( + nextAttempt, + now, + opts?.random, + ) + : null; + const baseSchedule = computedBaseSchedule + ? { ...computedBaseSchedule, maxAttempts } + : null; + const transientRecovery = + retryReason === BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON + ? readTransientRecoveryContractFromRun(run) + : null; + const codexTransientFallbackMode = + agent.adapterType === "codex_local" && + transientRecovery?.errorFamily === "transient_upstream" + ? resolveCodexTransientFallbackMode(nextAttempt) + : null; + const transientRetryNotBefore = transientRecovery?.retryNotBefore ?? null; + const contextSnapshot = parseObject(run.contextSnapshot); + // A retry inherits the durable authorization scope of its source run. Do + // not promote an untrusted or legacy contextSnapshot.issueId into a new + // issue binding: that could either violate the FK or misclassify history. + const issueId = run.scopeKind === "issue" ? run.issueId : null; + + if (!baseSchedule) { + const exhaustion = { + retryReason, + scheduledRetryAttempt: consumedAttempts, + maxAttempts, + }; + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: `Bounded retry exhausted after ${consumedAttempts} scheduled attempts; no further automatic retry will be queued`, + payload: exhaustion, + retryExhaustion: exhaustion, + }); + if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) { + await escalatePlanApprovalResumeFailureNeedsAttention({ + run, + issueId, + attempt: Math.min( + consumedAttempts, + maxAttempts, + ), + maxAttempts, + }).catch((error) => { + logger.warn( + { err: error, runId: run.id, issueId }, + "failed to escalate exhausted plan-approval resume failure", + ); + }); + } + return { + outcome: "retry_exhausted" as const, + attempt: nextAttempt, + maxAttempts, + }; + } + + if (await legacyExecutionNeedsReconciliationWithEvidence(db, run)) { + return { + outcome: "not_scheduled" as const, + reason: + "Reconcile the previous execution before retrying; safe provider recovery is unavailable.", + errorCode: "legacy_execution_requires_reconciliation" as const, + issueId: readNonEmptyString(run.contextSnapshot?.issueId), + }; + } + if (retryReason !== MAX_TURN_CONTINUATION_RETRY_REASON) { + const invokability = await getAgentInvokability(agent); + if (!invokability.invokable) { + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: + "Scheduled retry suppressed because the agent is not invokable", + payload: { + retryReason, + scheduledRetryAttempt: nextAttempt, + maxAttempts, + reason: invokability.reason, + invalidOrgChain: invokability.invalidOrgChain, + ...invokability.details, + }, + }); + return { + outcome: "not_scheduled" as const, + reason: + "Scheduled retry suppressed because the agent is not invokable", + errorCode: "agent_not_invokable" as const, + issueId, + }; + } + } + + const schedule = + transientRetryNotBefore && + transientRetryNotBefore.getTime() > baseSchedule.dueAt.getTime() + ? { + ...baseSchedule, + dueAt: transientRetryNotBefore, + delayMs: Math.max( + 0, + transientRetryNotBefore.getTime() - now.getTime(), + ), + } + : baseSchedule; + + const requiresIssueGate = + isTransientWorkspaceGitScanCode(run.errorCode) || + hasConversationContinuationPolicy(run.resultJson) || + (retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) || + retryReason === MAX_TURN_CONTINUATION_RETRY_REASON || + retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON; + if (requiresIssueGate) { + const gate = await runDispatch.evaluateScheduledRetryGate({ + runId: run.id, + companyId: run.companyId, + retryReasonOverride: retryReason, + now, + }); + if (!gate.allowed) { + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: gate.reason, + payload: { + retryReason, + scheduledRetryAttempt: nextAttempt, + maxAttempts, + ...gate.details, + }, + }); + return { + outcome: "not_scheduled" as const, + reason: gate.reason, + errorCode: gate.errorCode, + issueId: gate.issueId, + }; + } + } + const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null); + const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey); + const interactionContinuationPayload = + retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON + ? { + mutation: "interaction", + interactionId: readNonEmptyString(contextSnapshot.interactionId), + interactionKind: readNonEmptyString( + contextSnapshot.interactionKind, + ), + interactionStatus: readNonEmptyString( + contextSnapshot.interactionStatus, + ), + continuationPolicy: readNonEmptyString( + contextSnapshot.continuationPolicy, + ), + } + : {}; + const workspaceValidationRetryPayload = + retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON && + isWorkspaceValidationFailedRun(run) + ? readWorkspaceValidationPayloadFromRun(run) + : null; + const shouldQuarantineWorkspaceForRetry = + workspaceValidationRetryPayload !== null && + Object.keys(workspaceValidationRetryPayload).length > 0; + const retryContextSnapshot: Record = withRecoveryContext( + { + ...contextSnapshot, + executionRetryAccounting: accountingForScheduledRetry(run, retryReason, schedule.attempt), + retryOfRunId: run.id, + wakeReason, + retryReason, + ...(retryReason === WORKSPACE_BUSY_RETRY_REASON + ? { + failureRetriesBeforeWorkspaceWait: + executionFailureRetryCount(run), + } + : {}), + ...((retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) + ? { failureRetriesBeforeAiConnectionWait: executionFailureRetryCount(run) } + : {}), + ...(shouldQuarantineWorkspaceForRetry + ? { + workspaceValidationRecovery: { + strategy: "quarantine_failed_workspace_and_retry_clean", + sourceRunId: run.id, + reason: + readNonEmptyString(workspaceValidationRetryPayload?.reason) ?? + WORKSPACE_VALIDATION_FAILURE_CODE, + fingerprint: readNonEmptyString( + workspaceValidationRetryPayload?.fingerprint, + ), + failedExecutionWorkspaceId: readNonEmptyString( + workspaceValidationRetryPayload?.executionWorkspaceId, + ), + }, + } + : {}), + ...(transientRecovery + ? { errorFamily: transientRecovery.errorFamily } + : {}), + scheduledRetryAttempt: schedule.attempt, + scheduledRetryAt: schedule.dueAt.toISOString(), + ...(transientRetryNotBefore + ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } + : {}), + ...(transientRecovery?.errorFamily === "provider_quota" && + transientRetryNotBefore + ? { + providerQuotaRetryNotBefore: + transientRetryNotBefore.toISOString(), + } + : {}), + ...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}), + }, + "normal_model", + ); + // A successor resolves current assignments; the failed run retains its snapshot. + delete retryContextSnapshot.skillVersions; + delete retryContextSnapshot.skillSelectionSnapshot; + delete retryContextSnapshot.shadowedSkills; + const responsibleUserId = await resolveResponsibleUserIdForRunContext( + run, + retryContextSnapshot, + ); + const continuationRetryIdempotencyKey = + retryReason === MAX_TURN_CONTINUATION_RETRY_REASON + ? `max-turn-continuation:${run.companyId}:${issueId ?? "no-issue"}:${run.id}:${schedule.attempt}` + : retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON + ? `interaction-continuation:${run.companyId}:${issueId ?? "no-issue"}:${run.id}:${schedule.attempt}` + : null; + + type ScheduledRetryTransactionResult = + | { + outcome: "scheduled"; + run: typeof heartbeatRuns.$inferSelect; + reusedExisting: boolean; + } + | { + outcome: "not_scheduled"; + reason: string; + errorCode: + | "issue_not_found" + | "issue_reassigned" + | "issue_cancelled" + | "issue_terminal_status" + | "issue_not_in_progress" + | "continuation_user_authorization_missing" + | "issue_execution_lock_changed"; + issueId: string | null; + details: Record; + }; + + const scheduleResult = await db.transaction( + async (tx): Promise => { + // All automatic failure paths share the same predecessor claim. A + // duplicate monitor, restart sweep or wake must reuse its successor. + if ( + retryReason !== MAX_TURN_CONTINUATION_RETRY_REASON && + retryReason !== INTERACTION_CONTINUATION_INFRA_RETRY_REASON + ) { + if (issueId) + await tx.execute( + sql`select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`, + ); + await tx.execute( + sql`select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`, + ); + const [existing] = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.retryOfRunId, run.id), + ), + ) + .limit(1); + if (existing) + return { + outcome: "scheduled", + run: existing, + reusedExisting: true, + }; + } + if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) { + if (issueId) { + await tx.execute( + sql`select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`, + ); + } else { + await tx.execute( + sql`select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`, + ); + } + + const existingContinuation = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.retryOfRunId, run.id), + eq(heartbeatRuns.scheduledRetryReason, retryReason), + eq(heartbeatRuns.scheduledRetryAttempt, schedule.attempt), + inArray(heartbeatRuns.status, [ + ...MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES, + ]), + issueId + ? sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}` + : sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' is null`, + ), + ) + .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) + .limit(1) + .then((rows) => rows[0] ?? null); + + if (existingContinuation) { + if (existingContinuation.wakeupRequestId) { + const existingWakeup = await tx + .select({ coalescedCount: agentWakeupRequests.coalescedCount }) + .from(agentWakeupRequests) + .where( + eq( + agentWakeupRequests.id, + existingContinuation.wakeupRequestId, + ), + ) + .then((rows) => rows[0] ?? null); + + await tx + .update(agentWakeupRequests) + .set({ + coalescedCount: (existingWakeup?.coalescedCount ?? 0) + 1, + updatedAt: now, + }) + .where( + eq( + agentWakeupRequests.id, + existingContinuation.wakeupRequestId, + ), + ); + } + + return { + outcome: "scheduled", + run: existingContinuation, + reusedExisting: true, + }; + } + } + + if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON) { + if (issueId) { + await tx.execute( + sql`select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`, + ); + } else { + await tx.execute( + sql`select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`, + ); + } + + const existingContinuation = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.retryOfRunId, run.id), + eq(heartbeatRuns.scheduledRetryReason, retryReason), + eq(heartbeatRuns.scheduledRetryAttempt, schedule.attempt), + inArray(heartbeatRuns.status, [ + ...MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES, + ]), + issueId + ? sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}` + : sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' is null`, + ), + ) + .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) + .limit(1) + .then((rows) => rows[0] ?? null); + + if (existingContinuation) { + if (existingContinuation.wakeupRequestId) { + const existingWakeup = await tx + .select({ coalescedCount: agentWakeupRequests.coalescedCount }) + .from(agentWakeupRequests) + .where( + eq( + agentWakeupRequests.id, + existingContinuation.wakeupRequestId, + ), + ) + .then((rows) => rows[0] ?? null); + + await tx + .update(agentWakeupRequests) + .set({ + coalescedCount: (existingWakeup?.coalescedCount ?? 0) + 1, + updatedAt: now, + }) + .where( + eq( + agentWakeupRequests.id, + existingContinuation.wakeupRequestId, + ), + ); + } + + return { + outcome: "scheduled", + run: existingContinuation, + reusedExisting: true, + }; + } + + if (issueId) { + const lockedIssue = await tx + .select({ + id: issues.id, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + executionRunId: issues.executionRunId, + }) + .from(issues) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, run.companyId), + ), + ) + .then((rows) => rows[0] ?? null); + + if (!lockedIssue) { + return { + outcome: "not_scheduled", + reason: + "Scheduled max-turn continuation suppressed because the target issue no longer exists", + errorCode: "issue_not_found", + issueId, + details: { issueId }, + }; + } + + if (lockedIssue.assigneeAgentId !== run.agentId) { + return { + outcome: "not_scheduled", + reason: + "Scheduled max-turn continuation suppressed because issue ownership changed", + errorCode: "issue_reassigned", + issueId, + details: { + issueId, + previousAssigneeAgentId: run.agentId, + currentAssigneeAgentId: lockedIssue.assigneeAgentId, + }, + }; + } + + if ( + lockedIssue.status === "cancelled" || + lockedIssue.status === "done" + ) { + return { + outcome: "not_scheduled", + reason: `Scheduled max-turn continuation suppressed because issue reached terminal status (${lockedIssue.status})`, + errorCode: + lockedIssue.status === "cancelled" + ? "issue_cancelled" + : "issue_terminal_status", + issueId, + details: { issueId, currentStatus: lockedIssue.status }, + }; + } + + if (lockedIssue.status !== "in_progress") { + return { + outcome: "not_scheduled", + reason: `Scheduled max-turn continuation suppressed because issue is no longer in_progress (current status: ${lockedIssue.status})`, + errorCode: "issue_not_in_progress", + issueId, + details: { + issueId, + currentStatus: lockedIssue.status, + requiredStatus: "in_progress", + }, + }; + } + + if (lockedIssue.executionRunId !== run.id) { + return { + outcome: "not_scheduled", + reason: + "Scheduled max-turn continuation suppressed because the issue execution lock belongs to a different run", + errorCode: "issue_execution_lock_changed", + issueId, + details: { + issueId, + expectedExecutionRunId: run.id, + currentExecutionRunId: lockedIssue.executionRunId, + }, + }; + } + } + } + + if ( + (retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) && issueId && + !isNonAssigneeWorkspaceBusyRetry(retryReason, contextSnapshot) + ) { + // The issue row is locked above. Recheck after the preflight gate so + // cancellation or recovery cannot leave a successor without its lock. + const [lockedIssue] = await tx.select({ executionRunId: issues.executionRunId }) + .from(issues).where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))); + if (lockedIssue?.executionRunId !== run.id) { + return { + outcome: "not_scheduled", issueId, errorCode: "issue_execution_lock_changed", + reason: "Subscription retry suppressed because the task execution lock changed", + details: { issueId, expectedExecutionRunId: run.id, currentExecutionRunId: lockedIssue?.executionRunId ?? null }, + }; + } + } + + const scheduledRunId = randomUUID(); + if (contextSnapshot.explicitUserContinuation) { + const continuation = issueId && retryReason === "transient_failure" ? await admitExplicitContinuationRetry({ + db: tx as unknown as Db, companyId: run.companyId, issueId, agentId: run.agentId, + parentRunId: run.id, successorRunId: scheduledRunId, now, + }) : null; + if (!continuation) return { + outcome: "not_scheduled", issueId, + errorCode: "continuation_user_authorization_missing", + reason: "The automatic retry could not revalidate the original user continuation.", + details: {}, + }; + retryContextSnapshot.explicitUserContinuation = continuation; + retryContextSnapshot.previousRunId = continuation.previousRunId; + } + + await notifyRunDispatch(tx, { companyId: run.companyId }); + const wakeupRequest = await tx + .insert(agentWakeupRequests) + .values({ + companyId: run.companyId, + agentId: run.agentId, + source: "automation", + triggerDetail: "system", + reason: wakeReason, + payload: withRecoveryContext( + { + ...(issueId ? { issueId } : {}), + retryOfRunId: run.id, + ...interactionContinuationPayload, + retryReason, + ...(transientRecovery + ? { errorFamily: transientRecovery.errorFamily } + : {}), + scheduledRetryAttempt: schedule.attempt, + scheduledRetryAt: schedule.dueAt.toISOString(), + ...(transientRetryNotBefore + ? { + transientRetryNotBefore: + transientRetryNotBefore.toISOString(), + } + : {}), + ...(transientRecovery?.errorFamily === "provider_quota" && + transientRetryNotBefore + ? { + providerQuotaRetryNotBefore: + transientRetryNotBefore.toISOString(), + } + : {}), + ...(codexTransientFallbackMode + ? { codexTransientFallbackMode } + : {}), + }, + "normal_model", + ), + status: "queued", + requestedByActorType: "system", + requestedByActorId: null, + idempotencyKey: continuationRetryIdempotencyKey, + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + const scheduledRun = await tx + .insert(heartbeatRuns) + .values({ + id: scheduledRunId, + companyId: run.companyId, + agentId: run.agentId, + scopeKind: run.scopeKind, + issueId, + invocationSource: "automation", + triggerDetail: "system", + status: "scheduled_retry", + wakeupRequestId: wakeupRequest.id, + contextSnapshot: retryContextSnapshot, + ...(hasConversationContinuationPolicy(run.resultJson) + ? { resultJson: { conversationContinuation: CONVERSATION_CONTINUATION_POLICY } } : {}), + responsibleUserId, + sessionIdBefore: sessionBefore, + retryOfRunId: run.id, + scheduledRetryAt: schedule.dueAt, + scheduledRetryAttempt: schedule.attempt, + scheduledRetryReason: retryReason, + continuationAttempt: readContinuationAttempt( + retryContextSnapshot.livenessContinuationAttempt, + ), + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + await tx + .update(agentWakeupRequests) + .set({ + runId: scheduledRun.id, + updatedAt: now, + }) + .where(eq(agentWakeupRequests.id, wakeupRequest.id)); + + let detachWorkspaceFromIssue = false; + if (issueId && shouldQuarantineWorkspaceForRetry) { + const issueWorkspace = await tx + .select({ + id: issues.id, + companyId: issues.companyId, + executionWorkspaceId: issues.executionWorkspaceId, + }) + .from(issues) + .where( + and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)), + ) + .for("update") + .then((rows) => rows[0] ?? null); + const failedExecutionWorkspaceId = + readNonEmptyString( + workspaceValidationRetryPayload?.executionWorkspaceId, + ) ?? readNonEmptyString(issueWorkspace?.executionWorkspaceId); + + if (issueWorkspace && failedExecutionWorkspaceId) { + const failedWorkspace = await tx + .select({ + id: executionWorkspaces.id, + companyId: executionWorkspaces.companyId, + sourceIssueId: executionWorkspaces.sourceIssueId, + status: executionWorkspaces.status, + metadata: executionWorkspaces.metadata, + }) + .from(executionWorkspaces) + .where( + and( + eq(executionWorkspaces.id, failedExecutionWorkspaceId), + eq(executionWorkspaces.companyId, run.companyId), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null); + + const workspaceBelongsToIssue = failedWorkspace + ? failedWorkspace.sourceIssueId === issueId + : false; + + if ( + failedWorkspace && + workspaceBelongsToIssue && + issueWorkspace.executionWorkspaceId === failedExecutionWorkspaceId + ) { + const existingMetadata = parseObject(failedWorkspace.metadata); + const quarantine = { + reason: WORKSPACE_VALIDATION_FAILURE_CODE, + retryReason, + sourceRunId: run.id, + retryRunId: scheduledRun.id, + issueId, + sourceIssueId: failedWorkspace.sourceIssueId ?? null, + quarantinedAt: now.toISOString(), + workspaceValidation: workspaceValidationRetryPayload ?? {}, + }; + await tx + .update(executionWorkspaces) + .set({ + status: "archived", + closedAt: now, + cleanupEligibleAt: null, + cleanupReason: WORKSPACE_VALIDATION_FAILURE_CODE, + metadata: { + ...existingMetadata, + workspaceValidationQuarantine: quarantine, + }, + updatedAt: now, + }) + .where( + and( + eq(executionWorkspaces.id, failedWorkspace.id), + eq(executionWorkspaces.companyId, run.companyId), + ), + ); + + await logActivity(tx as unknown as Db, { + companyId: run.companyId, + actorType: "system", + actorId: "heartbeat", + agentId: run.agentId, + runId: run.id, + action: "execution_workspace.workspace_validation_quarantined", + entityType: "execution_workspace", + entityId: failedWorkspace.id, + details: quarantine, + }); + detachWorkspaceFromIssue = + issueWorkspace.executionWorkspaceId === + failedExecutionWorkspaceId; + } + } + } + + if (issueId) { + await notifyStatusCardIssueChange(tx, run.companyId); + await tx + .update(issues) + .set({ + executionRunId: scheduledRun.id, + checkoutRunId: sql`case when ${issues.checkoutRunId} = ${run.id} then null else ${issues.checkoutRunId} end`, + executionAgentNameKey: normalizeAgentNameKey(agent.name), + executionLockedAt: now, + ...(detachWorkspaceFromIssue + ? { + executionWorkspaceId: null, + executionWorkspacePreference: null, + } + : {}), + updatedAt: now, + }) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, run.companyId), + eq(issues.executionRunId, run.id), + ), + ); + } + + return { + outcome: "scheduled", + run: scheduledRun, + reusedExisting: false, + }; + }, + ); + + if (scheduleResult.outcome === "not_scheduled") { + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: scheduleResult.reason, + payload: { + retryReason, + scheduledRetryAttempt: nextAttempt, + maxAttempts, + ...scheduleResult.details, + }, + }); + return { + outcome: "not_scheduled" as const, + reason: scheduleResult.reason, + errorCode: scheduleResult.errorCode, + issueId: scheduleResult.issueId, + }; + } + + const retryRun = scheduleResult.run; + const dueAt = retryRun.scheduledRetryAt + ? new Date(retryRun.scheduledRetryAt) + : schedule.dueAt; + + if (scheduleResult.reusedExisting) { + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "info", + message: `Reused existing continuation retry ${retryRun.scheduledRetryAttempt}/${schedule.maxAttempts}`, + payload: { + retryRunId: retryRun.id, + retryReason, + idempotencyKey: continuationRetryIdempotencyKey, + scheduledRetryAttempt: retryRun.scheduledRetryAttempt, + scheduledRetryAt: dueAt.toISOString(), + }, + }); + + return { + outcome: "scheduled" as const, + run: retryRun, + dueAt, + attempt: retryRun.scheduledRetryAttempt, + maxAttempts: schedule.maxAttempts, + reusedExisting: true, + }; + } + + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: `Scheduled bounded retry ${schedule.attempt}/${schedule.maxAttempts} for ${schedule.dueAt.toISOString()}`, + payload: { + retryRunId: retryRun.id, + retryReason, + ...(transientRecovery + ? { errorFamily: transientRecovery.errorFamily } + : {}), + scheduledRetryAttempt: schedule.attempt, + scheduledRetryAt: schedule.dueAt.toISOString(), + baseDelayMs: schedule.baseDelayMs, + delayMs: schedule.delayMs, + ...(transientRetryNotBefore + ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } + : {}), + ...(transientRecovery?.errorFamily === "provider_quota" && + transientRetryNotBefore + ? { + providerQuotaRetryNotBefore: + transientRetryNotBefore.toISOString(), + } + : {}), + ...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}), + }, + }); + + if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) { + await recordPlanApprovalResumeFailureRetry({ + run, + issueId, + retryRunId: retryRun.id, + attempt: schedule.attempt, + maxAttempts: schedule.maxAttempts, + }).catch((error) => { + logger.warn( + { err: error, runId: run.id, issueId, retryRunId: retryRun.id }, + "failed to record plan-approval resume retry failure", + ); + }); + } + + return { + outcome: "scheduled" as const, + run: retryRun, + dueAt, + attempt: schedule.attempt, + maxAttempts: schedule.maxAttempts, + }; + } + + // Finds a running heartbeat run (other than the caller's) whose context + // issue shares the same project workspace, i.e. the run that currently + // "holds" the shared working tree. Runs that have been silent past + // WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS do not count — a zombie holder must + // not park other work forever. This cutoff is independent of informational + // silence warnings. When isolated workspaces are enabled, holders whose + // issue explicitly opted into an isolated workspace never touch the shared + // tree, so they are excluded; a NULL/agent_default mode may resolve to the + // shared tree and counts as a holder (over-serializing is the safe + // direction). When the isolated-workspaces experiment is off, every run + // resolves to the shared tree, so no holder is excluded. + async function findSharedWorkspaceHolder(input: { + companyId: string; + projectWorkspaceId: string; + excludeIssueId: string; + excludeRunId: string; + honorIsolatedWorkspaceModes: boolean; + now?: Date; + }): Promise { + const staleCutoff = new Date( + (input.now ?? new Date()).getTime() - + WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS, + ); + return await db + .select({ + runId: heartbeatRuns.id, + agentId: heartbeatRuns.agentId, + issueId: sql`${issues.id}::text`, + issueIdentifier: issues.identifier, + }) + .from(heartbeatRuns) + .innerJoin( + issues, + and( + eq(issues.companyId, heartbeatRuns.companyId), + sql`${issues.id}::text = ${heartbeatRuns.contextSnapshot} ->> 'issueId'`, + ), + ) + .where( + and( + eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.status, "running"), + ne(heartbeatRuns.id, input.excludeRunId), + // Last observed activity: output beats start beats creation. A run + // that started recently but has not written output yet is live. + sql`coalesce(${heartbeatRuns.lastOutputAt}, ${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt}) >= ${staleCutoff.toISOString()}::timestamptz`, + eq(issues.projectWorkspaceId, input.projectWorkspaceId), + ne(sql`${issues.id}::text`, input.excludeIssueId), + ...(input.honorIsolatedWorkspaceModes + ? [ + or( + // Covers both a NULL settings blob and a blob without a mode + // key; either may still resolve to the shared workspace. + sql`${issues.executionWorkspaceSettings} ->> 'mode' is null`, + notInArray( + sql`${issues.executionWorkspaceSettings} ->> 'mode'`, + [...ISOLATED_EXECUTION_WORKSPACE_MODES], + ), + ), + ] + : []), + ), + ) + .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) + .limit(1) + .then((rows) => rows[0] ?? null); + } + + // Finds a running run (other than the caller's) on an issue bound to the + // same reused execution workspace, typically a parent run that is still + // active after delegating a child that inherited its workspace. Unlike + // findSharedWorkspaceHolder, a holder silent past + // WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS is still returned (flagged stale): + // silence does not prove the holder stopped, so the caller must not treat + // the folder as free. A live holder is preferred over a stale one. + async function findExecutionWorkspaceHolder(input: { + companyId: string; + executionWorkspaceId: string; + excludeRunId: string; + now?: Date; + }): Promise { + const staleCutoff = new Date( + (input.now ?? new Date()).getTime() - + WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS, + ); + const lastActivityBefore = sql`coalesce(${heartbeatRuns.lastOutputAt}, ${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt}) < ${staleCutoff.toISOString()}::timestamptz`; + return await db + .select({ + runId: heartbeatRuns.id, + agentId: heartbeatRuns.agentId, + issueId: sql`${issues.id}::text`, + issueIdentifier: issues.identifier, + stale: lastActivityBefore, + }) + .from(heartbeatRuns) + .innerJoin( + issues, + and( + eq(issues.companyId, heartbeatRuns.companyId), + eq(issues.executionWorkspaceId, input.executionWorkspaceId), + or( + eq(issues.executionRunId, heartbeatRuns.id), + sql`${issues.id}::text = coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId')`, + ), + ), + ) + .where( + and( + eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.status, "running"), + ne(heartbeatRuns.id, input.excludeRunId), + ), + ) + .orderBy(asc(lastActivityBefore), asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) + .limit(1) + .then((rows) => rows[0] ?? null); + } + + // Credential rotation can briefly contend with a fresh runtime read. Keep + // the task on its automatic pre-provider retry path while the lock clears. + async function finalizeAiConnectionBusyDeferral( + run: typeof heartbeatRuns.$inferSelect, + error: HttpError, + wasIssueAssignee: boolean, + ) { + const now = new Date(); + const cancelled = await setRunStatusIfRunning(run.id, "cancelled", { + error: error.message, errorCode: AI_CONNECTION_BUSY_RETRY_REASON, finishedAt: now, + resultJson: { + executionRecovery: { kind: "ai_connection_wait", providerWorkStarted: false }, + cancellation: { + source: "control_plane", + expected: true, + initiator: { type: "system" }, + reason: "Waiting for shared AI credentials", + recordedAt: now.toISOString(), + }, + }, + contextSnapshot: { + ...parseObject(run.contextSnapshot), + aiConnectionBusyDeferredWhileAssignee: wasIssueAssignee, + }, + }); + if (!cancelled.updated) return; + await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: now, error: error.message }).catch(() => undefined); + const cancelledRun = cancelled.run ?? await getRun(run.id); + const agent = await getAgent(run.agentId); + let scheduled = false; + try { + if (cancelledRun && agent) { + const retry = await scheduleBoundedRetryForRun(cancelledRun, agent, { + now, retryReason: AI_CONNECTION_BUSY_RETRY_REASON, wakeReason: "ai_connection_busy_retry", + maxAttempts: (cancelledRun.scheduledRetryAttempt ?? 0) + 1, + delayMs: computeWorkspaceBusyRetryDelayMs(), + }); + scheduled = retry.outcome === "scheduled"; + await appendRunEvent(cancelledRun, { + eventType: "lifecycle", stream: "system", level: "info", + message: scheduled ? "Waiting for the shared AI subscription. This task will retry automatically." : "The AI subscription is busy; this task can no longer retry automatically.", + payload: { retryScheduled: scheduled }, + }); + } + } finally { + try { + if (cancelledRun && !scheduled) await releaseIssueExecutionAndPromote(cancelledRun); + } finally { + await finalizeAgentStatus(run.agentId, "cancelled", null, { wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run) }); + } + } + } + + // Terminal handling for a WorkspaceBusyDeferral thrown by the pre-dispatch + // gate: cancel the run (contention is not a failure), schedule a + // workspace_busy retry, and leave the agent idle. The issue execution lock + // transfers to the scheduled retry run inside scheduleBoundedRetryForRun, so + // the issue keeps an active execution path and recovery leaves it alone. + // Deferral has no attempt ceiling — the retry keeps rescheduling while a + // live holder exists, and holder staleness (not a counter) is what prevents + // waiting on a zombie. If no retry could be scheduled (agent no longer + // invokable), the lock is released so the issue does not strand on a + // cancelled run. + async function finalizeWorkspaceBusyDeferral( + run: typeof heartbeatRuns.$inferSelect, + deferral: WorkspaceBusyDeferral, + ) { + const now = new Date(); + const cancelWrite = await setRunStatusIfRunning(run.id, "cancelled", { + error: deferral.message, + errorCode: WORKSPACE_BUSY_ERROR_CODE, + finishedAt: now, + resultJson: { + executionRecovery: { + kind: "workspace_wait", + providerWorkStarted: false, + }, + cancellation: { + source: "control_plane", + expected: true, + initiator: { type: "system" }, + reason: `Waiting for the shared ${deferral.executionWorkspaceId ? "execution" : "project"} workspace`, + recordedAt: now.toISOString(), + }, + workspaceBusy: { + projectWorkspaceId: deferral.projectWorkspaceId, + executionWorkspaceId: deferral.executionWorkspaceId, + holderRunId: deferral.holder.runId, + holderIssueId: deferral.holder.issueId, + deferralAttempt: deferral.deferralAttempt, + }, + }, + // Recorded on the run (and inherited by the scheduled retry's context) + // so the retry promotion gate can tell a non-assignee wake — where an + // assignee mismatch is the expected state — from a reassignment race. + contextSnapshot: { + ...parseObject(run.contextSnapshot), + workspaceBusyDeferredWhileAssignee: deferral.wasIssueAssignee, + }, + }); + if (!cancelWrite.updated) { + logger.info( + { runId: run.id, currentStatus: cancelWrite.run?.status ?? null }, + "skipping workspace-busy deferral finalization because the run already left running state", + ); + return; + } + await setWakeupStatus(run.wakeupRequestId, "cancelled", { + finishedAt: now, + error: deferral.message, + }).catch(() => undefined); + + const cancelledRun = + cancelWrite.run ?? (await getRun(run.id).catch(() => null)); + const agentRow = await getAgent(run.agentId).catch(() => null); + let scheduleOutcome: string | null = null; + if (cancelledRun && agentRow) { + const scheduleResult = await scheduleBoundedRetryForRun( + cancelledRun, + agentRow, + { + now, + retryReason: WORKSPACE_BUSY_RETRY_REASON, + wakeReason: WORKSPACE_BUSY_RETRY_WAKE_REASON, + // Always admit the next attempt: workspace-busy deferral is bounded by + // holder liveness, not by an attempt counter. + maxAttempts: (cancelledRun.scheduledRetryAttempt ?? 0) + 1, + delayMs: computeWorkspaceBusyRetryDelayMs(), + }, + ).catch((scheduleErr) => { + logger.error( + { err: scheduleErr, runId: run.id }, + "failed to schedule workspace-busy retry after deferral", + ); + return null; + }); + scheduleOutcome = scheduleResult?.outcome ?? null; + } + + if (cancelledRun) { + await appendRunEvent(cancelledRun, { + eventType: "lifecycle", + stream: "system", + level: "info", + message: + scheduleOutcome === "scheduled" + ? `Deferred: ${deferral.message}. Retry ${deferral.deferralAttempt + 1} scheduled; the run waits for the workspace to free.` + : `Deferred: ${deferral.message}. No retry could be scheduled; releasing the issue for other runs.`, + payload: { + projectWorkspaceId: deferral.projectWorkspaceId, + executionWorkspaceId: deferral.executionWorkspaceId, + holderRunId: deferral.holder.runId, + holderIssueId: deferral.holder.issueId, + deferralAttempt: deferral.deferralAttempt, + retryScheduled: scheduleOutcome === "scheduled", + }, + }).catch(() => undefined); + } + + if (cancelledRun && scheduleOutcome !== "scheduled") { + await releaseIssueExecutionAndPromote(cancelledRun).catch( + (releaseErr) => { + logger.error( + { err: releaseErr, runId: run.id }, + "failed to release issue execution after workspace-busy deferral", + ); + }, + ); + } + + await finalizeAgentStatus(run.agentId, "cancelled", null, { + wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), + }).catch(() => undefined); + } + + async function scheduleInteractionContinuationInfrastructureRetryIfEligible( + run: typeof heartbeatRuns.$inferSelect, + agent: typeof agents.$inferSelect, + ) { + if (await hasCaseHandoffForRun(run)) return null; + if (!run.wakeupRequestId) return null; + if (!isResolvedInteractionContinuationWakeContext(run.contextSnapshot)) + return null; + if (!isRetryableInteractionContinuationInfrastructureFailure(run)) { + const context = parseObject(run.contextSnapshot); + const issueId = readNonEmptyString(context.issueId); + await escalatePlanApprovalResumeFailureNeedsAttention({ + run, + issueId, + attempt: Math.min( + run.scheduledRetryAttempt ?? + INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, + INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, + ), + maxAttempts: INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, + }).catch((error) => { + logger.warn( + { err: error, runId: run.id, issueId }, + "failed to escalate non-retryable plan-approval resume failure", + ); + }); + return null; + } + + return scheduleBoundedRetryForRun(run, agent, { + retryReason: INTERACTION_CONTINUATION_INFRA_RETRY_REASON, + wakeReason: INTERACTION_CONTINUATION_INFRA_WAKE_REASON, + maxAttempts: INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, + }); + } + + async function nextScheduledRetryAt() { + return runDispatch.nextScheduledRetryAt({ cutoff: await getWorktreeExecutionCutoff() }); + } + + async function promoteDueScheduledRetries(now = new Date()) { + const cutoff = await getWorktreeExecutionCutoff(); + const result = await runDispatch.promoteDueScheduledRetries({ + now, + cutoff, + }); + applyRunDispatchPostCommitEffects(result.postCommitEffects); + return { promoted: result.promoted, runIds: result.runIds }; + } + + async function getIssueRetryRun( + companyId: string, + issueId: string, + statuses: Array<"scheduled_retry" | "queued" | "running" | "cancelled">, + ) { + if (statuses.length === 0) return null; + return db + .select({ + run: heartbeatRuns, + agentName: agents.name, + }) + .from(heartbeatRuns) + .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id)) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.status, statuses), + sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`, + sql`${heartbeatRuns.retryOfRunId} is not null`, + ), + ) + .orderBy( + desc(heartbeatRuns.updatedAt), + desc(heartbeatRuns.createdAt), + desc(heartbeatRuns.id), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + } + + function summarizeIssueScheduledRetryRun(row: { + run: typeof heartbeatRuns.$inferSelect; + agentName: string | null; + }) { + return { + runId: row.run.id, + status: row.run.status as + "scheduled_retry" | "queued" | "running" | "cancelled", + agentId: row.run.agentId, + agentName: row.agentName, + retryOfRunId: row.run.retryOfRunId, + scheduledRetryAt: row.run.scheduledRetryAt, + scheduledRetryAttempt: row.run.scheduledRetryAttempt, + scheduledRetryReason: row.run.scheduledRetryReason, + error: row.run.error, + errorCode: row.run.errorCode, + }; + } + + async function retryScheduledRetryNow(input: { + issueId: string; + actor?: { + actorType?: "user" | "agent" | "system"; + actorId?: string | null; + }; + now?: Date; + }) { + const now = input.now ?? new Date(); + const issue = await db + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, input.issueId)) + .then((rows) => rows[0] ?? null); + if (!issue) throw notFound("Issue not found"); + + const scheduled = await getIssueRetryRun(issue.companyId, issue.id, [ + "scheduled_retry", + ]); + if (!scheduled) { + const alreadyPromoted = await getIssueRetryRun( + issue.companyId, + issue.id, + ["queued", "running"], + ); + if (alreadyPromoted) { + return { + outcome: "already_promoted" as const, + message: "Scheduled retry was already promoted", + scheduledRetry: summarizeIssueScheduledRetryRun(alreadyPromoted), + }; + } + return { + outcome: "no_scheduled_retry" as const, + message: "No live scheduled retry exists for this issue", + scheduledRetry: null, + }; + } + + const contextSnapshot = { + ...parseObject(scheduled.run.contextSnapshot), + scheduledRetryAt: now.toISOString(), + retryNowRequestedAt: now.toISOString(), + retryNowRequestedByActorType: input.actor?.actorType ?? null, + retryNowRequestedByActorId: input.actor?.actorId ?? null, + }; + + const updated = await db.transaction(async (tx) => { + await notifyRunDispatch(tx, { companyId: scheduled.run.companyId }); + const row = await tx + .update(heartbeatRuns) + .set({ + scheduledRetryAt: now, + contextSnapshot, + updatedAt: now, + }) + .where( + and( + eq(heartbeatRuns.id, scheduled.run.id), + eq(heartbeatRuns.status, "scheduled_retry"), + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + if (!row) return null; + + if (row.wakeupRequestId) { + const wakeupPayload = { + ...parseObject( + await tx + .select({ payload: agentWakeupRequests.payload }) + .from(agentWakeupRequests) + .where(eq(agentWakeupRequests.id, row.wakeupRequestId)) + .then((rows) => rows[0]?.payload ?? null), + ), + scheduledRetryAt: now.toISOString(), + retryNowRequestedAt: now.toISOString(), + }; + await tx + .update(agentWakeupRequests) + .set({ + payload: wakeupPayload, + updatedAt: now, + }) + .where(eq(agentWakeupRequests.id, row.wakeupRequestId)); + } + + return row; + }); + + if (!updated) { + const alreadyPromoted = await getIssueRetryRun( + issue.companyId, + issue.id, + ["queued", "running"], + ); + if (alreadyPromoted) { + return { + outcome: "already_promoted" as const, + message: "Scheduled retry was already promoted", + scheduledRetry: summarizeIssueScheduledRetryRun(alreadyPromoted), + }; + } + return { + outcome: "no_scheduled_retry" as const, + message: "No live scheduled retry exists for this issue", + scheduledRetry: null, + }; + } + + await appendRunEvent(updated, { + eventType: "lifecycle", + stream: "system", + level: "info", + message: "Scheduled retry was requested to run now", + payload: { + issueId: issue.id, + scheduledRetryAttempt: updated.scheduledRetryAttempt, + scheduledRetryAt: updated.scheduledRetryAt + ? new Date(updated.scheduledRetryAt).toISOString() + : null, + scheduledRetryReason: updated.scheduledRetryReason, + requestedByActorType: input.actor?.actorType ?? null, + requestedByActorId: input.actor?.actorId ?? null, + }, + }); + + const promotion = await runDispatch.promoteScheduledRetry({ + runId: updated.id, + companyId: updated.companyId, + now, + }); + if (promotion.outcome === "promoted") { + applyRunDispatchPostCommitEffects(promotion.postCommitEffects); + } + // Promotion can preserve this row as a cleanup wait. Read that exact run, + // not an older cancelled retry or the pre-promotion schedule. + const currentRun = await getRun(updated.id); + const scheduledRetry = currentRun + ? summarizeIssueScheduledRetryRun({ + run: currentRun, + agentName: scheduled.agentName, + }) + : null; + + if (currentRun?.status === "scheduled_retry") { + return { + outcome: "waiting" as const, + message: parseObject(currentRun.resultJson?.executionWait).cause === "execution_owner_active" + ? "Waiting for execution cleanup. Paperclip will retry automatically once cleanup finishes." + : "The retry remains scheduled. Paperclip will check again at the scheduled time.", + scheduledRetry, + }; + } + + if (promotion.outcome === "promoted") { + return { + outcome: "promoted" as const, + message: "Scheduled retry was promoted to the queued run pool", + scheduledRetry, + }; + } + if (promotion.outcome === "gate_suppressed") { + return { + outcome: "gate_suppressed" as const, + message: promotion.reason, + scheduledRetry, + }; + } + if (currentRun && ["queued", "running"].includes(currentRun.status)) { + return { + outcome: "already_promoted" as const, + message: "Scheduled retry was already promoted", + scheduledRetry, + }; + } + return { + outcome: "no_scheduled_retry" as const, + message: "No live scheduled retry exists for this issue", + scheduledRetry: null, + }; + } + + function timerClaimWasFirstHeartbeat( + run: Pick, + ): true | undefined { + return parseObject(run.contextSnapshot).timerClaimWasFirstHeartbeat === true + ? true + : undefined; + } + + return { + scheduleBoundedRetryForRun, + timerClaimWasFirstHeartbeat, + scheduleInteractionContinuationInfrastructureRetryIfEligible, + findSharedWorkspaceHolder, + findExecutionWorkspaceHolder, + finalizeAiConnectionBusyDeferral, + finalizeWorkspaceBusyDeferral, + promoteDueScheduledRetries, + nextScheduledRetryAt, + retryScheduledRetryNow, + + scheduleBoundedRetry: async ( + runId: string, + opts?: { + now?: Date; + random?: () => number; + retryReason?: string; + wakeReason?: string; + maxAttempts?: number; + delayMs?: number; + }, + ) => { + const run = await getRun(runId, { unsafeFullResultJson: true }); + if (!run) return { outcome: "missing_run" as const }; + const agent = await getAgent(run.agentId); + if (!agent) return { outcome: "missing_agent" as const }; + return scheduleBoundedRetryForRun(run, agent, opts); + }, + }; +} diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.ts index bc795417faa..8a1f850e02c 100644 --- a/server/src/services/instance-settings.ts +++ b/server/src/services/instance-settings.ts @@ -1 +1,671 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; +import { notifySkillVersionRetentionReconstruction } from "./skill-version-retention-notifications.js"; +import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js"; +import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js"; +import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; +import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; +import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; +import { notifyNativeResultCommit } from "./native-result-commit-notifications.js"; +import { notifyNativeWorkspaceFinalization } from "./native-workspace-finalization-notifications.js"; +import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js"; +import { notifyInstructionCollection } from "./instruction-collection-notifications.js"; +import { notifyDirectoryCleanup } from "./directory-cleanup-notifications.js"; +import { notifyInstructionCommit } from "./instruction-commit-notifications.js"; +import { notifyScheduledRetryCancellation } from "./scheduled-retry-cancellation-notifications.js"; +import { notifyVoiceChange } from "./voice/voice-recovery-notifications.js"; +import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +import type { Db } from "@paperclipai/db"; +import { companies, instanceSettings } from "@paperclipai/db"; + +/** + * A `Db` or an open transaction handle — the subset of query builders the + * settings writes use. Lets `update` run inside a caller's transaction so + * it commits atomically with a sibling write. + */ +type InstanceSettingsTransaction = Parameters[0]>[0]; +export type InstanceSettingsWriteDb = Pick< + Db | InstanceSettingsTransaction, + "select" | "insert" | "update" +>; +import { + DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE, + DEFAULT_BACKUP_RETENTION, + PAPERCLIP_CLOUD_MANAGED_BY, + instanceGeneralSettingsSchema, + type InstanceGeneralSettings, + instanceExperimentalSettingsSchema, + type InstanceExperimentalSettings, + type InstanceExperimentalSettingsWithManaged, + type ManagedExperimentalFeatureKey, + type ManagedSettingMetadata, + type PatchInstanceGeneralSettings, + type InstanceSettings, + type PatchInstanceSettings, + type PatchInstanceExperimentalSettings, +} from "@paperclipai/shared"; +import { + INSTANCE_FEATURE_CATALOG, + applyOperatorGeneralDefaults, + stripOperatorGeneralEchoes, +} from "@paperclipai/shared"; +import { eq } from "drizzle-orm"; +import { getManagedInstanceConfig, type ManagedInstanceConfig } from "./managed-config.js"; +import { getOperatorSettingDefaults } from "./setting-defaults.js"; +import { isCloudStagingExperimentalSetting, isCloudStagingInstance } from "./cloud-instance.js"; +import { forbidden } from "../errors.js"; + +const DEFAULT_SINGLETON_KEY = "default"; +const instanceGeneralSettingsStorageSchema = instanceGeneralSettingsSchema.strip(); +const instanceExperimentalSettingsStorageSchema = instanceExperimentalSettingsSchema.strip(); +const TRUTHY_RUNTIME_ENV_VALUES = new Set(["1", "true", "yes", "on"]); + +interface InstanceSettingsServiceOptions { + runtimeEnv?: Record; + now?: () => Date; +} + +type WorktreeRunExecutionSuppressedReason = + | "not_worktree_runtime" + | "flag_disabled" + | "missing_cutoff" + | "missing_instance_id" + | "instance_id_mismatch" + | "settings_read_error"; + +export type WorktreeRunExecutionActivationState = + | { + armed: true; + cutoff: string; + activationInstanceId: string; + reason: null; + } + | { + armed: false; + cutoff: null; + activationInstanceId: string | null; + reason: WorktreeRunExecutionSuppressedReason; + }; + +export function isTruthyRuntimeEnvValue(value: string | undefined) { + return typeof value === "string" && TRUTHY_RUNTIME_ENV_VALUES.has(value.trim().toLowerCase()); +} + +function getRuntimeInstanceId(env: Record) { + const instanceId = env.PAPERCLIP_INSTANCE_ID?.trim(); + return instanceId ? instanceId : null; +} + +function stripServerManagedExperimentalPatchFields( + patch: PatchInstanceExperimentalSettings | Record, +): PatchInstanceExperimentalSettings { + const { + worktreeRunExecutionActivatedAt: _ignoredActivatedAt, + worktreeRunExecutionActivationInstanceId: _ignoredActivationInstanceId, + ...patchable + } = patch as Record; + return patchable as PatchInstanceExperimentalSettings; +} + +export function applyExperimentalSettingsPatch( + current: unknown, + patch: PatchInstanceExperimentalSettings | Record, + options: InstanceSettingsServiceOptions = {}, +): InstanceExperimentalSettings { + const previousExperimental = normalizeExperimentalSettings(current); + const patchable = stripServerManagedExperimentalPatchFields(patch); + const nextExperimental = normalizeExperimentalSettings({ + ...previousExperimental, + ...patchable, + }); + const hasWorktreeRunExecutionPatch = Object.prototype.hasOwnProperty.call( + patchable, + "enableWorktreeRunExecution", + ); + + if (!hasWorktreeRunExecutionPatch) { + return nextExperimental; + } + + if (nextExperimental.enableWorktreeRunExecution !== true) { + return { + ...nextExperimental, + worktreeRunExecutionActivatedAt: null, + worktreeRunExecutionActivationInstanceId: null, + }; + } + + if (previousExperimental.enableWorktreeRunExecution === true) { + return nextExperimental; + } + + const runtimeEnv = options.runtimeEnv ?? process.env; + if (!isTruthyRuntimeEnvValue(runtimeEnv.PAPERCLIP_IN_WORKTREE)) { + return nextExperimental; + } + + return { + ...nextExperimental, + worktreeRunExecutionActivatedAt: (options.now ?? (() => new Date()))().toISOString(), + worktreeRunExecutionActivationInstanceId: getRuntimeInstanceId(runtimeEnv), + }; +} + +function suppressWorktreeRunExecution( + reason: WorktreeRunExecutionSuppressedReason, + activationInstanceId: string | null = null, +): WorktreeRunExecutionActivationState { + return { + armed: false, + cutoff: null, + activationInstanceId, + reason, + }; +} + +export function resolveWorktreeRunExecutionActivation( + experimental: InstanceExperimentalSettings, + currentInstanceId: string | null | undefined, +): WorktreeRunExecutionActivationState { + if (experimental.enableWorktreeRunExecution !== true) { + return suppressWorktreeRunExecution( + "flag_disabled", + experimental.worktreeRunExecutionActivationInstanceId, + ); + } + if (!experimental.worktreeRunExecutionActivatedAt) { + return suppressWorktreeRunExecution( + "missing_cutoff", + experimental.worktreeRunExecutionActivationInstanceId, + ); + } + if (!currentInstanceId) { + return suppressWorktreeRunExecution( + "missing_instance_id", + experimental.worktreeRunExecutionActivationInstanceId, + ); + } + if (experimental.worktreeRunExecutionActivationInstanceId !== currentInstanceId) { + return suppressWorktreeRunExecution( + "instance_id_mismatch", + experimental.worktreeRunExecutionActivationInstanceId, + ); + } + return { + armed: true, + cutoff: experimental.worktreeRunExecutionActivatedAt, + activationInstanceId: currentInstanceId, + reason: null, + }; +} + +export async function resolveWorktreeRunExecutionActivationState(options: { + getExperimental: () => Promise; + runtimeEnv?: Record; +}): Promise { + const runtimeEnv = options.runtimeEnv ?? process.env; + if (!isTruthyRuntimeEnvValue(runtimeEnv.PAPERCLIP_IN_WORKTREE)) { + return suppressWorktreeRunExecution("not_worktree_runtime"); + } + try { + return resolveWorktreeRunExecutionActivation( + await options.getExperimental(), + getRuntimeInstanceId(runtimeEnv), + ); + } catch { + return suppressWorktreeRunExecution("settings_read_error"); + } +} + +function normalizeGeneralSettings(raw: unknown): InstanceGeneralSettings { + const parsed = instanceGeneralSettingsStorageSchema.safeParse(raw ?? {}); + if (parsed.success) { + return { + censorUsernameInLogs: parsed.data.censorUsernameInLogs ?? false, + feedbackDataSharingPreference: + parsed.data.feedbackDataSharingPreference ?? DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE, + backupRetention: parsed.data.backupRetention ?? DEFAULT_BACKUP_RETENTION, + // Absent => unrestricted; only carry through an explicit policy. + ...(parsed.data.executionMode ? { executionMode: parsed.data.executionMode } : {}), + }; + } + return { + censorUsernameInLogs: false, + feedbackDataSharingPreference: DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE, + backupRetention: DEFAULT_BACKUP_RETENTION, + }; +} + +export function normalizeExperimentalSettings(raw: unknown): InstanceExperimentalSettings { + const parsed = instanceExperimentalSettingsStorageSchema.safeParse(raw ?? {}); + if (parsed.success) { + return { + enableEnvironments: parsed.data.enableEnvironments ?? false, + enableNativeRunner: parsed.data.enableNativeRunner ?? true, + enableAiConnectionRouters: parsed.data.enableAiConnectionRouters ?? false, + enableManagedSandboxOnly: parsed.data.enableManagedSandboxOnly ?? false, + enableIsolatedWorkspaces: parsed.data.enableIsolatedWorkspaces ?? false, + enableIsolatedWorkspacesByDefault: parsed.data.enableIsolatedWorkspacesByDefault ?? false, + enableStreamlinedLeftNavigation: parsed.data.enableStreamlinedLeftNavigation ?? true, + enableStreamlinedUi: parsed.data.enableStreamlinedUi ?? true, + // Apps graduated from Experimental. Ignore historical off values while + // continuing to accept the compatibility key in stored settings. + enableApps: true, + enableMcpAggregators: true, + enablePublicMcp: parsed.data.enablePublicMcp ?? false, + enableOpenAiDot: parsed.data.enableOpenAiDot ?? false, + enableMuse: parsed.data.enableMuse ?? false, + enableGrokBot: parsed.data.enableGrokBot ?? false, + enableChatConnectors: parsed.data.enableChatConnectors ?? false, + enableGitHubReviewBots: parsed.data.enableGitHubReviewBots ?? false, + enableMemoryConnectors: parsed.data.enableMemoryConnectors ?? false, + enablePaperclipNotebook: parsed.data.enablePaperclipNotebook ?? false, + enableCases: parsed.data.enableCases ?? false, + enableAgentChat: parsed.data.enableAgentChat ?? false, + enableCombinedInboxTasks: parsed.data.enableCombinedInboxTasks ?? false, + enableArtifactsV1: parsed.data.enableArtifactsV1 ?? false, + enableConferenceRoomChat: parsed.data.enableConferenceRoomChat ?? false, + enableClassicTaskInterface: parsed.data.enableClassicTaskInterface ?? false, + enableIssuePlanDecompositions: parsed.data.enableIssuePlanDecompositions ?? false, + enableExperimentalFileViewer: parsed.data.enableExperimentalFileViewer ?? false, + enableExternalObjects: parsed.data.enableExternalObjects ?? false, + enableSmokeLab: parsed.data.enableSmokeLab ?? false, + enableBuiltInAgents: parsed.data.enableBuiltInAgents ?? false, + enableBetaSkills: parsed.data.enableBetaSkills ?? false, + enableSummaries: parsed.data.enableSummaries ?? false, + enableStatusCards: parsed.data.enableStatusCards ?? false, + enableDecisions: parsed.data.enableDecisions ?? false, + enableFastResponses: parsed.data.enableFastResponses ?? false, + enableGoalsSidebarLink: parsed.data.enableGoalsSidebarLink ?? false, + enableServerInfoDebugView: parsed.data.enableServerInfoDebugView ?? false, + enablePaperclipDeveloperMode: parsed.data.enablePaperclipDeveloperMode ?? false, + enableSimplifiedEnglishInteractions: parsed.data.enableSimplifiedEnglishInteractions ?? false, + enableFirstTaskPlanProposal: parsed.data.enableFirstTaskPlanProposal ?? false, + autoRestartDevServerWhenIdle: parsed.data.autoRestartDevServerWhenIdle ?? false, + enableWorkspaceBranchReconcileForward: parsed.data.enableWorkspaceBranchReconcileForward ?? true, + enableWorkspaceDirtyQuarantineRepair: parsed.data.enableWorkspaceDirtyQuarantineRepair ?? true, + enableOwnerInstanceAdmin: parsed.data.enableOwnerInstanceAdmin ?? false, + enableSandboxDuplexBridge: parsed.data.enableSandboxDuplexBridge ?? false, + enableRunnerPreviewIngress: parsed.data.enableRunnerPreviewIngress ?? false, + enableWorktreeRunExecution: parsed.data.enableWorktreeRunExecution ?? false, + worktreeRunExecutionActivatedAt: parsed.data.worktreeRunExecutionActivatedAt ?? null, + worktreeRunExecutionActivationInstanceId: + parsed.data.worktreeRunExecutionActivationInstanceId ?? null, + }; + } + return { + enableEnvironments: false, + enableNativeRunner: true, + enableAiConnectionRouters: false, + enableManagedSandboxOnly: false, + enableIsolatedWorkspaces: false, + enableIsolatedWorkspacesByDefault: false, + enableStreamlinedLeftNavigation: true, + enableStreamlinedUi: true, + enableApps: true, + enableMcpAggregators: true, + enablePublicMcp: false, + enableOpenAiDot: false, + enableMuse: false, + enableGrokBot: false, + enableChatConnectors: false, + enableGitHubReviewBots: false, + enableMemoryConnectors: false, + enablePaperclipNotebook: false, + enableCases: false, + enableAgentChat: false, + enableCombinedInboxTasks: false, + enableArtifactsV1: false, + enableConferenceRoomChat: false, + enableClassicTaskInterface: false, + enableIssuePlanDecompositions: false, + enableExperimentalFileViewer: false, + enableExternalObjects: false, + enableSmokeLab: false, + enableBuiltInAgents: false, + enableBetaSkills: false, + enableSummaries: false, + enableStatusCards: false, + enableDecisions: false, + enableFastResponses: false, + enableGoalsSidebarLink: false, + enableServerInfoDebugView: false, + enablePaperclipDeveloperMode: false, + enableSimplifiedEnglishInteractions: false, + enableFirstTaskPlanProposal: false, + autoRestartDevServerWhenIdle: false, + enableWorkspaceBranchReconcileForward: true, + enableWorkspaceDirtyQuarantineRepair: true, + enableOwnerInstanceAdmin: false, + enableSandboxDuplexBridge: false, + enableRunnerPreviewIngress: false, + enableWorktreeRunExecution: false, + worktreeRunExecutionActivatedAt: null, + worktreeRunExecutionActivationInstanceId: null, + }; +} + +export type ManagedExperimentalKeyMetadata = Partial< + Record +>; + +/** + * Overlay the cloud managed-config feature values over normalized settings. + * + * Read-time precedence: code floor (cloud) > managed overlay > tenant DB + * value > schema default. (No code floors are expressed as flags today — + * floors are enforced in code at the guarded routes, independent of any + * flag value.) The overlay is deliberately never persisted: it re-asserts on + * every read, so a DB restore or manual row edit cannot resurrect a + * capability the harness has disabled. + */ +export function applyManagedExperimentalOverlay( + experimental: InstanceExperimentalSettings, + managedConfig: ManagedInstanceConfig | null, +): { experimental: InstanceExperimentalSettings; managedKeys: ManagedExperimentalKeyMetadata } { + if (!managedConfig) return { experimental, managedKeys: {} }; + const next: InstanceExperimentalSettings = { ...experimental }; + const managedKeys: ManagedExperimentalKeyMetadata = {}; + for (const [key, value] of Object.entries(managedConfig.features) as Array< + [ManagedExperimentalFeatureKey, boolean] + >) { + // Existing Cloud stack configs may still carry retired flags. Accept the + // document during rollout, but never let retired flags disable Apps or MCP aggregators. + if (key === "enableApps" || key === "enableMcpAggregators") continue; + next[key] = value; + managedKeys[key] = { managed: true, managedBy: PAPERCLIP_CLOUD_MANAGED_BY }; + } + return { experimental: next, managedKeys }; +} + +/** + * Keep self-hosted-only defaults out of Cloud. + * + * The experimental schema carries one default per flag, and the feature + * catalog pins it to `selfHostedDefault`. A flag that is on by default for + * self-hosted but off by default for Cloud (`selfHostedDefault: true`, + * `cloudDefault: false`) would therefore normalize to "on" for a managed + * instance whose tenant row and managed overlay both leave it unset. Re-assert + * the declared Cloud default for exactly those flags. An explicit tenant value + * or a managed feature value still wins (the overlay is applied afterwards). + */ +export function applyCloudCatalogDefaults( + experimental: InstanceExperimentalSettings, + rawStored: unknown, + managedConfig: ManagedInstanceConfig | null, +): InstanceExperimentalSettings { + if (!managedConfig) return experimental; + const stored = + rawStored && typeof rawStored === "object" && !Array.isArray(rawStored) + ? (rawStored as Record) + : {}; + const next: InstanceExperimentalSettings = { ...experimental }; + for (const [key, entry] of Object.entries(INSTANCE_FEATURE_CATALOG)) { + if (entry.cloudDefault !== false || entry.selfHostedDefault !== true) continue; + if (typeof stored[key] === "boolean") continue; + if (typeof managedConfig.features[key as ManagedExperimentalFeatureKey] === "boolean") continue; + (next as unknown as Record)[key] = false; + } + return next; +} + +/** + * Keep the write path from freezing a self-hosted default into a Cloud row. + * + * `updateExperimental` persists the whole normalized object, and the schema + * normalizes an omitted flag to its self-hosted default. Without this step an + * unrelated experimental write (say, turning on Cases) would store + * `enableNativeRunner: true` on a managed instance whose tenant row had never + * mentioned the flag; every later read would then treat the stored boolean as + * an explicit tenant choice and stop re-asserting the Cloud default. + * + * For each guarded flag (see `applyCloudCatalogDefaults`), the stored key is + * left absent unless the tenant already stored a boolean or this patch sets + * the flag to something other than the Cloud default. A patch value equal to + * the Cloud default is a full-GET echo of the read-time overlay, not a + * choice, and is stripped the same way `stripOperatorGeneralEchoes` treats + * operator defaults. Self-hosted rows are returned untouched. + */ +export function stripCloudCatalogDefaultEchoes( + rawStored: unknown, + patch: PatchInstanceExperimentalSettings | Record, + next: InstanceExperimentalSettings, + managedConfig: ManagedInstanceConfig | null, +): Partial { + if (!managedConfig) return next; + const stored = + rawStored && typeof rawStored === "object" && !Array.isArray(rawStored) + ? (rawStored as Record) + : {}; + const patchRecord = patch as Record; + const result: Record = { ...next }; + for (const [key, entry] of Object.entries(INSTANCE_FEATURE_CATALOG)) { + if (entry.cloudDefault !== false || entry.selfHostedDefault !== true) continue; + if (typeof stored[key] === "boolean") continue; + if ( + Object.prototype.hasOwnProperty.call(patchRecord, key) && + typeof patchRecord[key] === "boolean" && + patchRecord[key] !== entry.cloudDefault + ) { + continue; + } + delete result[key]; + } + return result as Partial; +} + +export function instanceSettingsService(db: Db, options: InstanceSettingsServiceOptions = {}) { + // Fail closed: a malformed PAPERCLIP_MANAGED_CONFIG throws here (and at + // boot in index.ts) rather than silently running without the overlay. + const managedConfig = getManagedInstanceConfig(options.runtimeEnv ?? process.env); + const staging = isCloudStagingInstance(options.runtimeEnv ?? process.env); + const enforcedManagedConfig = staging && managedConfig + ? { + ...managedConfig, + features: Object.fromEntries(Object.entries(managedConfig.features) + .filter(([key, enabled]) => enabled || !isCloudStagingExperimentalSetting(key))), + } + : managedConfig; + // Same posture for PAPERCLIP_SETTING_DEFAULTS: parsed once, applied per + // read, never persisted (see applyOperatorGeneralDefaults) — including on + // the write path, where a full-GET echo of the overlaid value is stripped + // back to the schema default (see stripOperatorGeneralEchoes). + const operatorDefaults = getOperatorSettingDefaults(options.runtimeEnv ?? process.env); + + function toGeneralView(raw: unknown): InstanceGeneralSettings { + return applyOperatorGeneralDefaults(normalizeGeneralSettings(raw), operatorDefaults); + } + + function toExperimentalView(raw: unknown): InstanceExperimentalSettingsWithManaged { + const stored = raw && typeof raw === "object" && !Array.isArray(raw) ? raw : {}; + const defaults = staging ? { ...managedConfig?.features, ...stored } : raw; + const { experimental, managedKeys } = applyManagedExperimentalOverlay( + applyCloudCatalogDefaults(normalizeExperimentalSettings(defaults), defaults, managedConfig), + enforcedManagedConfig, + ); + // Self-hosted responses stay byte-identical: no managedKeys field at all. + return managedConfig ? { ...experimental, managedKeys } : experimental; + } + + function toInstanceSettings(row: typeof instanceSettings.$inferSelect): InstanceSettings { + return { + id: row.id, + defaultEnvironmentId: row.defaultEnvironmentId ?? null, + general: toGeneralView(row.general), + experimental: toExperimentalView(row.experimental), + createdAt: row.createdAt, + updatedAt: row.updatedAt, + } as InstanceSettings; + } + async function getOrCreateRow(runner: InstanceSettingsWriteDb = db) { + const existing = await runner + .select() + .from(instanceSettings) + .where(eq(instanceSettings.singletonKey, DEFAULT_SINGLETON_KEY)) + .then((rows) => rows[0] ?? null); + if (existing) return existing; + + const now = new Date(); + const [created] = await runner + .insert(instanceSettings) + .values({ + singletonKey: DEFAULT_SINGLETON_KEY, + general: {}, + experimental: {}, + createdAt: now, + updatedAt: now, + }) + .onConflictDoUpdate({ + target: [instanceSettings.singletonKey], + set: { + updatedAt: now, + }, + }) + .returning(); + + if (created) return created; + + const raced = await runner + .select() + .from(instanceSettings) + .where(eq(instanceSettings.singletonKey, DEFAULT_SINGLETON_KEY)) + .then((rows) => rows[0] ?? null); + if (raced) return raced; + + throw new Error("Failed to initialize instance settings row"); + } + + return { + get: async (): Promise => toInstanceSettings(await getOrCreateRow()), + + update: async ( + patch: PatchInstanceSettings, + writeOptions?: { db?: InstanceSettingsWriteDb }, + ): Promise => { + // The write may run inside a caller-supplied transaction so it commits + // atomically with a sibling write (e.g. clearing the managed-default + // stamp on the environment row alongside a defaultEnvironmentId + // change). Reads use the same runner so the row is visible to the tx. + const runner = writeOptions?.db ?? db; + const current = await getOrCreateRow(runner); + const now = new Date(); + const [updated] = await runner + .update(instanceSettings) + .set({ + ...(Object.prototype.hasOwnProperty.call(patch, "defaultEnvironmentId") + ? { defaultEnvironmentId: patch.defaultEnvironmentId ?? null } + : {}), + updatedAt: now, + }) + .where(eq(instanceSettings.id, current.id)) + .returning(); + return toInstanceSettings(updated ?? current); + }, + + getGeneral: async ( + readOptions?: { db?: InstanceSettingsWriteDb }, + ): Promise => { + const row = await getOrCreateRow(readOptions?.db); + return toGeneralView(row.general); + }, + + getExperimental: async (): Promise => { + const row = await getOrCreateRow(); + return toExperimentalView(row.experimental); + }, + + updateGeneral: async (patch: PatchInstanceGeneralSettings): Promise => { + const current = await getOrCreateRow(); + const storedGeneral = normalizeGeneralSettings(current.general); + // A full-GET echo carries the overlaid operator value for a field the + // user never chose; stripping it keeps the overlay strictly read-time, + // so changing or unsetting the variable later still takes effect. + const nextGeneral = stripOperatorGeneralEchoes( + storedGeneral, + normalizeGeneralSettings({ ...storedGeneral, ...patch }), + operatorDefaults, + ); + const now = new Date(); + const [updated] = await db + .update(instanceSettings) + .set({ + general: { ...nextGeneral }, + updatedAt: now, + }) + .where(eq(instanceSettings.id, current.id)) + .returning(); + return toInstanceSettings(updated ?? current); + }, + + updateExperimental: async (patch: PatchInstanceExperimentalSettings): Promise => { + for (const [key, enabled] of Object.entries(managedConfig?.features ?? {})) { + if (enabled && patch[key as ManagedExperimentalFeatureKey] === false) { + throw forbidden(`${key} is enabled by Paperclip Cloud and cannot be disabled on this instance`, { + code: "settings_operator_managed", + }); + } + } + const current = await getOrCreateRow(); + // Guarded Cloud flags stay absent from the row unless chosen, so the + // read-time catalog default keeps applying (see stripCloudCatalogDefaultEchoes). + const nextExperimental = staging + ? applyExperimentalSettingsPatch(toExperimentalView(current.experimental), patch, options) + : stripCloudCatalogDefaultEchoes( + current.experimental, + patch, + applyExperimentalSettingsPatch(current.experimental, patch, options), + managedConfig, + ); + if (staging) { + const stored = current.experimental as Record | null; + for (const key of Object.keys(INSTANCE_FEATURE_CATALOG) as ManagedExperimentalFeatureKey[]) { + if (typeof stored?.[key] !== "boolean" && !Object.prototype.hasOwnProperty.call(patch, key)) { + delete nextExperimental[key]; + } + } + } + const now = new Date(); + const [updated] = await db.transaction(async tx => { + if (patch.enableStatusCards !== undefined) await notifyStatusCardEvaluation(tx); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.mcpEvents, "*"); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, "*"); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.grokDispatch); + if (patch.enableWorktreeRunExecution !== undefined) await notifyScheduledRetryCancellation(tx); + if (patch.enableWorktreeRunExecution !== undefined) { + await notifyRunDispatch(tx); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyExternalAgentSettlement(tx); + await notifyInstructionCommit(tx); + await notifyDirectoryCleanup(tx); + await notifySkillVersionRetentionReconstruction(tx); + await notifyInstructionCollection(tx); + await notifyCancelledWorkspacePreservation(tx); + await notifyExternalConversationSettlement(tx); + await notifyNativeRecoveryNotices(tx); + await notifyNativeRunProjection(tx); + await notifyNativeEnvironmentSettlement(tx); + await notifyNativeResultCommit(tx); + await notifyNativeWorkspaceFinalization(tx); + } + if (patch.enableChatConnectors !== undefined) await notifyVoiceChange(tx); + await notifyToolConnectionHealth(tx, "*"); + return tx + .update(instanceSettings) + .set({ + experimental: { ...nextExperimental }, + updatedAt: now, + }) + .where(eq(instanceSettings.id, current.id)) + .returning(); + }); + return toInstanceSettings(updated ?? current); + }, + + listCompanyIds: async (): Promise => + db + .select({ id: companies.id }) + .from(companies) + .then((rows) => rows.map((row) => row.id)), + }; +} diff --git a/server/src/services/issue-review-state-notifications.ts b/server/src/services/issue-review-state-notifications.ts index 82d51d88c3b..b0f98f61b2a 100644 --- a/server/src/services/issue-review-state-notifications.ts +++ b/server/src/services/issue-review-state-notifications.ts @@ -1 +1,15 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; +import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; +import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; + +export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) { + await notifyRunDispatch(tx, scope); await notifyStatusCardIssueChange(tx, scope.companyId); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyExternalConversationSettlement(tx, scope); + await notifyNativeReviewResponse(tx, scope); + await notifyNativeStatusReassessment(tx, scope); +} diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 0eff1c83613..faddf33dc93 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1 +1,14120 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; +import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; +import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; +import { notifyNativeResultCommit } from "./native-result-commit-notifications.js"; +import { notifyNativeWorkspaceFinalization } from "./native-workspace-finalization-notifications.js"; +import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; +import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; +import { notifyChatResponseAvailability } from "./chat-response-availability.js"; +import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js"; +import { notifyChatRunMilestones } from "./chat-run-milestone-notifications.js"; +import { notifyApprovalWork } from "./approval-work-notifications.js"; +import { notifyDecisionTargetWork, notifyDecisionTargetIssueRemoval } from "./decision-targets.js"; +import { assertOnboardingFirstTaskCreationAllowed, assertOnboardingFirstTaskDependenciesAllowed } from "./onboarding-first-task-admission.js"; +import { resolveConnectionIntentContinuationSource, resolveItemVerdictContinuationWake } from "./chat-interaction-continuation.js"; +import { insertIssueComment } from "./issue-comment-writes.js"; +import { measureRunTransaction } from "./execution-transaction-timing.js"; +import { readNativeMonitorSource } from "./native-runtime/native-monitor-delivery.js"; +import { readNativeContinuationSource } from "./native-runtime/native-continuation-delivery.js"; +import { assertNoCaseHandoffMutation, cancelCaseHandoffsForIssue, completeTerminalCaseHandoffsForIssue } from "./heartbeat/case-handoff-lifecycle.js"; +import { measureRunPhase } from "./execution-preparation-timing.js"; +import { automaticReadRecoveryCleanupCondition, directHumanCommentCondition, humanCreatedIssueCondition, pendingUserRequestCondition, pendingUserRequestRowCondition, touchedByUserCondition, userParticipationActivityCondition } from "./issue-participation.js"; +import { resolveProjectDefaultWorkspaceId } from "./project-default-workspace.js"; +import { type AuthorizationActor, executionWorkspaceReadSqlCondition, projectReadSqlCondition } from "./authorization.js"; +import { assertExecutionWorkspaceRecordAccess, assertTaskWorkspaceSourceProjectAccess } from "./task-workspace-access.js"; +import { type TaskWorkspaceSelection, type TaskWorkspaceIntent } from "@paperclipai/shared"; +import { executionWorkspaceService } from "./execution-workspaces.js"; +import { notifyChatPublicationWork } from "./chat-work-notifications.js"; +import { notifyTerminalWorkspaceWork } from "./terminal-workspace-work.js"; +import { requestRestorePointDeletion } from "./git-restore-point-ledger.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +import { recordChatHandoff, recordAcceptedChatHandoff, lockAcceptedChatHandoffSource, recordChatCompletion, existingChatCompletionReply, acknowledgeChatCompletionReply } from "./chat-completion-delivery.js"; +import { mirrorSlackBoardComment, slackBoardReplyBindings } from "./slack-board-messages.js"; +import { assertAgentRunWriteAllowed } from "../agent-run-cancellation.js"; +import { assertLegacyRunTaskOwnership } from "../legacy-run-task-ownership.js"; +import { retryIdempotentDatabaseOperation } from "../database-retry.js"; +import { externalConversationStateSql, nonIdleSlackIssueCondition, resumeSlackConversation } from "./slack-conversation-state.js"; +import { documentService } from "./documents.js"; +import { parseTaskSearch, taskSearchCtes, taskSearchScore } from "./task-search.js"; +import { createdFromIssueCondition } from "./issue-creation-origin.js"; +import { executionProjectionsForRuns } from "./execution-projection.js"; +import type { ExecutionProjection } from "@paperclipai/shared"; +import { Buffer } from "node:buffer"; +import { isDeepStrictEqual } from "node:util"; +import { createHash, randomUUID } from "node:crypto"; +import { isExplicitContinuationRetryClaim } from "./explicit-continuation-retry-claim.js"; +import { markdownToPlainText, parseMarkdown } from "chat"; +import { listExternalReviewWaits } from "./native-runtime/external-review-wait.js"; +import { + and, + asc, + desc, + eq, + gt, + getTableColumns, + gte, + inArray, + isNotNull, + isNull, + like, + lt, + lte, + ne, + notExists, + notInArray, + or, + sql, + type SQL, +} from "drizzle-orm"; +import { unionAll } from "drizzle-orm/pg-core"; +import type { Db } from "@paperclipai/db"; +import { + activityLog, + chatActions, + chatConversations, + chatDeliveries, + chatEndpoints, + chatMessageLinks, + chatPublications, + agentWakeupRequests, + agents, + authUsers, + approvals, + assets, + companies, + companyMemberships, + documentRevisions, + documents, + goals, + heartbeatRuns, + routineRuns, + executionWorkspaces, + issueApprovals, + issueAccessGrants, + issueAttachments, + issueCreateIdempotencyKeys, + issueInboxArchives, + issueLabels, + issueWatchdogs, + issuePlanDecompositions, + issueRecoveryActions, + issueRelations, + issueComments, + issueDocuments, + issueQuestionResponseDeliveries, + issueWorkProducts, + issueReadStates, + issueThreadInteractions, + toolActionRequests, + toolActionDeliveries, + toolInvocations, + issues, + labels, + projectWorkspaces, + projects, + toolConnections, + workspaceOperations, +} from "@paperclipai/db"; +import type { + AcceptedPlanDecomposition, + IssueComment, + IssueCommentAuthorType, + IssueCommentDerivedAuthorSource, + IssueCommentMetadata, + IssueCommentPresentation, + IssueBlockerAttention, + IssueReviewAttention, + IssueReviewAttentionPath, + IssueBlockedInboxAttention, + IssueBlockedInboxIssueRef, + IssueRelationIssueSummary, + IssueWatchdogSummary, + LowTrustBoundary, + SuccessfulRunHandoffState, +} from "@paperclipai/shared"; +import { + clampIssueRequestDepth, + extractAgentMentionIds, + extractProjectMentionIds, + issueCommentAuthorTypeSchema, + issueCommentMetadataSchema, + issueCommentPresentationSchema, + isAgentCreatedIssue, + isUuidLike, + normalizeIssueIdentifier as normalizeIssueReferenceIdentifier, +} from "@paperclipai/shared"; +import { conflict, HttpError, notFound, unprocessable } from "../errors.js"; +import { isForeignKeyViolation } from "../db-errors.js"; +import { logger } from "../middleware/logger.js"; +import { parseObject } from "../adapters/utils.js"; +import { + hydrateSuccessfulRunHandoffLiveness, + SUCCESSFUL_RUN_HANDOFF_LIVE_WAKE_STATUSES, +} from "./successful-run-handoff-state.js"; +import { + defaultIssueExecutionWorkspaceSettingsForProject, + gateProjectExecutionWorkspacePolicy, + issueExecutionWorkspaceModeForPersistedWorkspace, + isUnrunnableWorktreeCombo, + parseIssueExecutionWorkspaceSettings, + parseProjectExecutionWorkspacePolicy, + resolvePinnedIssueWorkspaceStrategyType, + WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, + WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE, + WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION, + type ParsedExecutionWorkspaceMode, +} from "./execution-workspace-policy.js"; +import { mergeExecutionWorkspaceConfig } from "./execution-workspaces.js"; +import { hasChatRunOwnedProviderInteraction } from "./chat-interaction-arbitration.js"; +import { readChatControlChronology } from "./chat-control-chronology.js"; +import { authorizeChatConversationForBoundRun } from "./native-runtime/chat-attachment-reuse.js"; +import { + assertDurableChatWakeupReceipt, + createDurableChatWakeupRequest, +} from "./durable-chat-wakeup.js"; +import { + authorizeNativeChatReviewPresentation, + retryNativeChatReviewPresentation, +} from "./native-runtime/native-chat-review-presentation.js"; +import { + applyIssueExecutionPolicyTransition, + buildInitialIssueMonitorFields, + normalizeIssueExecutionPolicy, +} from "./issue-execution-policy.js"; +import { assertQualityReviewPolicyMutation, assertQualityReviewPolicyReuse, assertRequiredQualityReviewComplete, requiredQualityReview } from "./required-quality-review.js"; +import { instanceSettingsService } from "./instance-settings.js"; +import { redactCurrentUserText } from "../log-redaction.js"; +import { redactSensitiveText } from "../redaction.js"; +import { + resolveIssueGoalId, + resolveNextIssueGoalId, +} from "./issue-goal-fallback.js"; +import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js"; +import { recoverSettledNativeRunLog } from "./native-runtime/native-run-log-recovery.js"; +import { getDefaultCompanyGoal } from "./goals.js"; +import { assertAssignableAgent } from "./agent-assignability.js"; +import { + CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON, + isExternalChatPresentationContext, + LEGACY_WITHHELD_RUN_COMMENT, + projectHistoricalHeartbeatRunComment, +} from "./heartbeat-run-summary.js"; +import { + DEFAULT_INSERT_CHUNK_ROWS, + insertRowsInChunks, +} from "./batch-insert.js"; +import type { + ImportIssueRow, + ImportIssueCommentRow, + ImportIssueAttachmentRow, +} from "./import-write-types.js"; +import { + summarizeIssueWatchdog, + upsertIssueWatchdogForIssue, +} from "./task-watchdogs.js"; +import { + isVerifiedIssueTreeControlInteractionWake, + issueTreeControlService, + type ActiveIssueTreePauseHoldGate, +} from "./issue-tree-control.js"; +import { + parseIssueGraphLivenessIncidentKey, + RECOVERY_ORIGIN_KINDS, +} from "./recovery/origins.js"; +import { + classifyIssueGraphLiveness, + classifyIssueReviewPaths, + type IssueGraphLivenessInput, + type IssueLivenessFinding, +} from "./recovery/issue-graph-liveness.js"; +import { visibleIssueCondition } from "./issue-visibility.js"; +import { finalizeStatusCardsForStalledGeneration } from "./status-card-finalization.js"; +import { finalizeSummarySlotsForTerminalIssue } from "./summary-slot-finalization.js"; +import { + logActivity, + persistActivity, + publishActivity, + type ActivityPublication, +} from "./activity-log.js"; +import { buildIssueChanges } from "./issue-change-receipt.js"; +import { ensurePersonalPrivateProject } from "./projects.js"; +import { projectSafeChatPublication } from "./chat-publication-projection.js"; +import { issueThreadInteractionAttentionAgentAllowed } from "./issue-thread-interaction-resolution.js"; + +const ALL_ISSUE_STATUSES = [ + "backlog", + "todo", + "in_progress", + "in_review", + "blocked", + "done", + "cancelled", +]; +const MAX_ISSUE_COMMENT_PAGE_LIMIT = 500; +const MAX_CHAT_PRESENTATION_ATTACHMENTS = 20; +export const ISSUE_LIST_DEFAULT_LIMIT = 500; +export const ISSUE_LIST_MAX_LIMIT = 1000; +export const ISSUE_BLOCKER_DIAGNOSTICS_MAX_BLOCKERS = 100; +export const ISSUE_WAKE_DIAGNOSTICS_MAX_WAKE_REQUESTS = 50; +export const ISSUE_WAKE_DIAGNOSTICS_MAX_ACTIVITY_RECORDS = 50; +export const ISSUE_WAKE_DIAGNOSTICS_LOOKBACK_DAYS = 14; +export const ISSUE_SUBTREE_DIAGNOSTICS_MAX_DEPTH = 8; +export const ISSUE_SUBTREE_DIAGNOSTICS_MAX_NODES = 100; +export const ISSUE_SUBTREE_DIAGNOSTICS_MAX_BLOCKERS_PER_NODE = 20; +export const ISSUE_SUBTREE_DIAGNOSTICS_MAX_WAKE_REQUESTS_PER_NODE = 5; +export const ISSUE_SUBTREE_DIAGNOSTICS_MAX_ACTIVITY_RECORDS_PER_NODE = 5; +const ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE = 500; +export const MAX_CHILD_ISSUES_CREATED_BY_HELPER = 25; +const MAX_CHILD_COMPLETION_SUMMARIES = 20; +const CHILD_COMPLETION_SUMMARY_BODY_MAX_CHARS = 500; +// Non-human author sentinels that agents post under. These ARE eligible for +// agent-attribution derivation even though `local-board` is also materialized +// as a row in the `user` table (it is the implicit board admin). Genuine human +// users — real signups with their own ids — are never reattributed. +const NON_HUMAN_SENTINEL_AUTHOR_USER_IDS = new Set(["local-board"]); +const ISSUE_COMMENT_RUN_LOG_DERIVATION_MAX_LOG_BYTES = 2_000_000; +const ISSUE_COMMENT_RUN_LOG_DERIVATION_CHUNK_BYTES = 256_000; +const ISSUE_COMMENT_RUN_LOG_DERIVATION_END_SLACK_MS = 60_000; +const ISSUE_COMMENT_RUN_LOG_DERIVATION_MAX_PARALLEL_READS = 8; +const ISSUE_COMMENT_RUN_LOG_DERIVATION_TIMEOUT_MS = 3_000; +export const ISSUE_CREATE_IDEMPOTENCY_KEY_RETENTION_DAYS = 7; +const ISSUE_CREATE_IDEMPOTENCY_KEY_RETENTION_MS = + ISSUE_CREATE_IDEMPOTENCY_KEY_RETENTION_DAYS * 24 * 60 * 60 * 1000; +const ISSUE_CREATE_IDEMPOTENCY_KEY_CLEANUP_BATCH_SIZE = 500; +const DELETED_ISSUE_COMMENT_BODY = ""; +const ISSUE_WAKE_DIAGNOSTICS_ACTIVITY_ACTIONS = [ + "issue.tree_hold_wakeup_deferred", +] as const; + +export type IssuePostCommitAction = { + type: "cancel_native_question_run"; + runId: string; + issueId: string; + issueStatus: string; +}; + +/** Execute side effects that must never run before the issue transaction commits. */ +export async function executeIssuePostCommitActions( + db: Db, + actions: readonly IssuePostCommitAction[], +): Promise { + if (actions.length === 0) return; + const { heartbeatService } = await import("./heartbeat.js"); + const heartbeat = heartbeatService(db); + const cancelledRunIds = new Set(); + for (const action of actions) { + if (cancelledRunIds.has(action.runId)) continue; + cancelledRunIds.add(action.runId); + try { + await heartbeat.cancelRun( + action.runId, + "Task closed while waiting for operator input", + { + resultJson: { + cancelledByIssueStatus: action.issueStatus, + cancelledIssueId: action.issueId, + }, + }, + ); + } catch (err) { + // The durable marker written by the issue transaction remains available + // to startup and periodic recovery. Do not report a post-commit failure + // as though the already-committed issue transition had rolled back. + logger.warn( + { err, runId: action.runId, issueId: action.issueId }, + "native question cancellation deferred to recovery sweep", + ); + } + } +} + +function wakeRequestTargetsIssue(issueId: string) { + return sql`( + ${agentWakeupRequests.payload} ->> 'issueId' = ${issueId} + or ${agentWakeupRequests.payload} ->> 'taskId' = ${issueId} + or ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'issueId' = ${issueId} + or ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'taskId' = ${issueId} + )`; +} + +function wakeDiagnosticActivityTargetsIssue(issueId: string) { + return sql`( + (${activityLog.entityType} = 'issue' and ${activityLog.entityId} = ${issueId}) + or ${activityLog.details} ->> 'issueId' = ${issueId} + or ${activityLog.details} ->> 'rootIssueId' = ${issueId} + )`; +} + +function assertTransition(from: string, to: string) { + if (from === to) return; + if (!ALL_ISSUE_STATUSES.includes(to)) { + throw conflict(`Unknown issue status: ${to}`); + } +} + +function applyStatusSideEffects( + status: string | undefined, + patch: Partial, +): Partial { + if (!status) return patch; + + if (status === "in_progress" && !patch.startedAt) { + patch.startedAt = new Date(); + } + if (status === "done") { + patch.completedAt = new Date(); + } + if (status === "cancelled") { + patch.cancelledAt = new Date(); + } + return patch; +} + +function workspaceWorktreeRequiresProjectDetails() { + return { + code: WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, + remediation: WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION, + }; +} + +function assertExplicitPinnedWorktreeIssueRunnable(input: { + projectId: string | null | undefined; + projectWorkspaceId: string | null | undefined; + executionWorkspaceId: string | null | undefined; + executionWorkspacePreference: string | null | undefined; + executionWorkspaceSettings: unknown; +}) { + const settings = parseIssueExecutionWorkspaceSettings( + input.executionWorkspaceSettings, + ); + const mode = settings?.mode; + if (mode !== "isolated_workspace" && mode !== "operator_branch") return; + + const resolvedMode = mode as ParsedExecutionWorkspaceMode; + if ( + isUnrunnableWorktreeCombo({ + issue: { + projectId: input.projectId ?? null, + projectWorkspaceId: input.projectWorkspaceId ?? null, + executionWorkspaceId: input.executionWorkspaceId ?? null, + executionWorkspacePreference: + input.executionWorkspacePreference ?? null, + }, + resolvedMode, + resolvedStrategy: resolvePinnedIssueWorkspaceStrategyType({ + mode: resolvedMode, + issueSettings: settings, + }), + hasResolvablePriorSessionWorkspace: false, + }) + ) { + throw unprocessable( + WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE, + workspaceWorktreeRequiresProjectDetails(), + ); + } +} + +function readStringFromRecord(record: unknown, key: string) { + if (!record || typeof record !== "object") return null; + const value = (record as Record)[key]; + return typeof value === "string" && value.trim().length > 0 + ? value.trim() + : null; +} + +async function resolveResponsibleUserIdForIssueCreate( + reader: DbReader, + companyId: string, + input: { + explicitResponsibleUserId?: string | null; + createdByUserId?: string | null; + parentId?: string | null; + originKind?: string | null; + originRunId?: string | null; + actorRunId?: string | null; + actorResponsibleUserId?: string | null; + trustExplicitResponsibleUserId?: boolean; + }, +) { + const explicitResponsibleUserId = readStringFromRecord( + input, + "explicitResponsibleUserId", + ); + if ( + explicitResponsibleUserId && + input.trustExplicitResponsibleUserId === true + ) + return explicitResponsibleUserId; + + if (input.originKind === "routine_execution" && input.originRunId) { + const routineRun = await reader + .select({ responsibleUserId: routineRuns.responsibleUserId }) + .from(routineRuns) + .where( + and( + eq(routineRuns.companyId, companyId), + eq(routineRuns.id, input.originRunId), + ), + ) + .then((rows) => rows[0] ?? null); + if (routineRun?.responsibleUserId) return routineRun.responsibleUserId; + } + + const actorResponsibleUserId = readStringFromRecord( + input, + "actorResponsibleUserId", + ); + if (actorResponsibleUserId) return actorResponsibleUserId; + + if (input.actorRunId) { + const actorRun = await reader + .select({ responsibleUserId: heartbeatRuns.responsibleUserId }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.id, input.actorRunId), + ), + ) + .then((rows) => rows[0] ?? null); + if (actorRun?.responsibleUserId) return actorRun.responsibleUserId; + } + + if (input.parentId) { + const parent = await reader + .select({ + responsibleUserId: issues.responsibleUserId, + createdByUserId: issues.createdByUserId, + }) + .from(issues) + .where( + and(eq(issues.companyId, companyId), eq(issues.id, input.parentId)), + ) + .then((rows) => rows[0] ?? null); + if (parent?.responsibleUserId) return parent.responsibleUserId; + if (parent?.createdByUserId) return parent.createdByUserId; + } + + return input.createdByUserId ?? null; +} + +function buildReusedExecutionWorkspaceConfigPatchFromIssueSettings( + settings: ReturnType, +) { + return { + environmentId: settings?.environmentId ?? null, + provisionCommand: settings?.workspaceStrategy?.provisionCommand ?? null, + runtimeProvisionCommand: + settings?.workspaceStrategy?.runtimeProvisionCommand ?? null, + teardownCommand: settings?.workspaceStrategy?.teardownCommand ?? null, + workspaceRuntime: settings?.workspaceRuntime ?? null, + }; +} + +// Accepted-plan children are not realized yet, so carry only unresolved +// workspace intent and let the first child run render/persist its own branch. +function buildPreRealizationExecutionWorkspaceSettings( + raw: unknown, +): Record | null { + const settings = parseIssueExecutionWorkspaceSettings(raw, { + includeEnvironmentId: true, + }); + if (!settings) return null; + const mode = + settings.mode && + settings.mode !== "inherit" && + settings.mode !== "reuse_existing" + ? settings.mode + : null; + const next: Record = {}; + if (mode) next.mode = mode; + if (settings.environmentId !== undefined) + next.environmentId = settings.environmentId; + if (settings.workspaceRuntime) + next.workspaceRuntime = settings.workspaceRuntime; + if (settings.workspaceStrategy) { + next.workspaceStrategy = { + type: settings.workspaceStrategy.type, + ...(settings.workspaceStrategy.baseRef + ? { baseRef: settings.workspaceStrategy.baseRef } + : {}), + ...(settings.workspaceStrategy.branchTemplate + ? { branchTemplate: settings.workspaceStrategy.branchTemplate } + : {}), + ...(settings.workspaceStrategy.existingBranch + ? { existingBranch: settings.workspaceStrategy.existingBranch } + : {}), + ...(settings.workspaceStrategy.worktreeParentDir + ? { worktreeParentDir: settings.workspaceStrategy.worktreeParentDir } + : {}), + ...(settings.workspaceStrategy.provisionCommand + ? { provisionCommand: settings.workspaceStrategy.provisionCommand } + : {}), + ...(settings.workspaceStrategy.runtimeProvisionCommand + ? { + runtimeProvisionCommand: + settings.workspaceStrategy.runtimeProvisionCommand, + } + : {}), + ...(settings.workspaceStrategy.teardownCommand + ? { teardownCommand: settings.workspaceStrategy.teardownCommand } + : {}), + }; + } + return Object.keys(next).length > 0 ? next : null; +} + +function toTimestampMs(value: Date | string | null | undefined) { + if (!value) return null; + const date = value instanceof Date ? value : new Date(value); + const timestamp = date.getTime(); + return Number.isFinite(timestamp) ? timestamp : null; +} + +type IssueCommentRunLogAttributionCandidate = { + id: string; + createdAt: Date | string; + authorAgentId?: string | null; + authorUserId?: string | null; + createdByRunId?: string | null; +}; + +type IssueCommentRunLogAttributionRun = { + runId: string; + agentId: string; + createdAt: Date | string; + startedAt?: Date | string | null; + finishedAt?: Date | string | null; + // Best-effort run log text. May be empty when logs were not read for a tier + // that does not need them (run-id / run-window-unique); only the + // `run_log_comment_post` tier consults this. + logContent: string; +}; + +type DerivedIssueCommentAttribution = { + derivedAuthorAgentId: string; + derivedCreatedByRunId: string; + derivedAuthorSource: IssueCommentDerivedAuthorSource; +}; + +/** + * Resolve a `created_by_run_id` safe for the heartbeat_runs FK; returns null for + * missing/invalid ids so an unknown run id never 500s a comment insert. + */ +async function resolveCommentCreatedByRun( + dbOrTx: any, + companyId: string, + runId: string | null | undefined, +): Promise<{ id: string; contextSnapshot: unknown } | null> { + const normalized = typeof runId === "string" ? runId.trim() : ""; + if (!normalized || !isUuidLike(normalized)) return null; + const existing = await dbOrTx + .select({ + id: heartbeatRuns.id, + contextSnapshot: heartbeatRuns.contextSnapshot, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, normalized), + eq(heartbeatRuns.companyId, companyId), + ), + ) + .then( + (rows: Array<{ id: string; contextSnapshot: unknown }>) => + rows[0] ?? null, + ); + return existing; +} + +export type ChatPublicationBinding = { + companyId: string; + conversationId: string; + endpointId: string; +}; + +async function resolvePublishedInteractionPromptBindings( + dbOrTx: any, + companyId: string, + issueId: string, + interactionId: string, + agentId: string, +): Promise { + const [publications, processedActions] = (await Promise.all([ + dbOrTx + .select({ + companyId: chatConversations.companyId, + conversationId: chatConversations.id, + endpointId: chatConversations.endpointId, + idempotencyKey: chatPublications.idempotencyKey, + payload: chatPublications.payload, + }) + .from(chatPublications) + .innerJoin( + chatConversations, + and( + eq(chatConversations.companyId, chatPublications.companyId), + eq(chatConversations.endpointId, chatPublications.endpointId), + eq(chatConversations.id, chatPublications.conversationId), + ), + ) + .innerJoin( + chatEndpoints, + and( + eq(chatEndpoints.companyId, chatConversations.companyId), + eq(chatEndpoints.id, chatConversations.endpointId), + ), + ) + .where( + and( + eq(chatPublications.companyId, companyId), + eq(chatPublications.issueId, issueId), + eq(chatPublications.state, "published"), + eq(chatConversations.issueId, issueId), + eq(chatEndpoints.assignedAgentId, agentId), + eq( + sql`${chatPublications.payload}->>'interactionId'`, + interactionId, + ), + notExists( + dbOrTx + .select({ id: chatPublications.id }) + .from(chatPublications) + .where( + and( + eq(chatPublications.conversationId, chatConversations.id), + eq(chatPublications.state, "published"), + or( + like(chatPublications.idempotencyKey, "control:new:%"), + like(chatPublications.idempotencyKey, "control:close:%"), + ), + ), + ), + ), + ), + ), + dbOrTx + .select({ + conversationId: chatActions.conversationId, + kind: chatActions.kind, + result: chatActions.result, + }) + .from(chatActions) + .where( + and( + eq(chatActions.companyId, companyId), + eq(chatActions.status, "processed"), + or( + inArray(chatActions.kind, [ + "question_answer", + "question_form_submit", + ]), + and( + eq(chatActions.kind, "confirmation_response"), + eq( + sql`${chatActions.result}->>'code'`, + "confirmation_resolution_committed_by_provider", + ), + ), + ), + eq( + sql`${chatActions.payload}->>'interactionId'`, + interactionId, + ), + ), + ), + ])) as [ + Array< + ChatPublicationBinding & { + idempotencyKey: string; + payload: { card?: { actions?: Array<{ type: string }> } }; + } + >, + Array<{ conversationId: string }>, + ]; + const providerWinnerConversationIds = new Set( + processedActions.map( + (action: { conversationId: string }) => action.conversationId, + ), + ); + const exactProviderWinner = providerWinnerConversationIds.size > 0; + const bindings = publications + .filter( + (publication) => + publication.idempotencyKey === + `interaction:${interactionId}:${publication.endpointId}` && + (!exactProviderWinner || + providerWinnerConversationIds.has(publication.conversationId)), + ) + .map((publication) => ({ + companyId: publication.companyId, + conversationId: publication.conversationId, + endpointId: publication.endpointId, + })); + return [ + ...new Map( + bindings.map((binding) => [binding.conversationId, binding]), + ).values(), + ]; +} + +function readChatWakeCommentIds( + contextSnapshot: Record, +): string[] { + const ids: string[] = []; + const append = (value: unknown) => { + if (typeof value !== "string") return; + const normalized = value.trim(); + if (!normalized || !isUuidLike(normalized) || ids.includes(normalized)) { + return; + } + ids.push(normalized); + }; + + const batched = contextSnapshot.wakeCommentIds; + if (Array.isArray(batched)) { + for (const value of batched) append(value); + } + append(contextSnapshot.wakeCommentId); + append(contextSnapshot.commentId); + return ids; +} + +/** A close retires its admitted sources, not every future source in a reusable + * provider thread. This exception requires a complete, independently admitted + * fresh batch; active state or copied run context alone never crosses a close. + * It grants presentation only. Provider dispatch retains its own current gate. */ +async function freshChatSourceAfterPublishedControl( + dbOrTx: any, + companyId: string, + issueId: string, + runId: string, + binding: ChatPublicationBinding, +): Promise { + const [run] = (await dbOrTx + .select() + .from(heartbeatRuns) + .where( + and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId)), + ) + .limit(1)) as Array; + const context = run?.contextSnapshot ?? {}; + const commentIds = readChatWakeCommentIds(context); + if ( + !run?.wakeupRequestId || + !run.startedAt || + !commentIds.length || + commentIds.length > 50 || + (run.nativeIssueId !== null && run.nativeIssueId !== issueId) || + (context.issueId ?? context.taskId) !== issueId + ) + return false; + const [owner] = (await dbOrTx + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.id, run.wakeupRequestId), + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.agentId, run.agentId), + eq(agentWakeupRequests.runId, run.id), + ), + ) + .limit(1)) as Array; + if (!owner || ["skipped", "cancelled"].includes(owner.status)) return false; + const admitted = (await dbOrTx + .select({ commentId: sql`${chatActions.payload}->>'commentId'` }) + .from(chatActions) + .innerJoin( + agentWakeupRequests, + and( + eq(agentWakeupRequests.id, chatActions.id), + eq(agentWakeupRequests.companyId, chatActions.companyId), + ), + ) + .where( + and( + eq(chatActions.companyId, companyId), + eq(chatActions.kind, "inbound_wakeup"), + or( + eq(agentWakeupRequests.runId, run.id), + eq(agentWakeupRequests.id, owner.id), + sql`${agentWakeupRequests.payload}->>'coalescedIntoWakeupRequestId' = ${owner.id}`, + ), + ), + ) + .limit(51)) as Array<{ commentId: string }>; + if ( + admitted.length !== commentIds.length || + admitted.some((row) => !commentIds.includes(row.commentId)) + ) + return false; + const [latestControl] = (await dbOrTx + .select({ publishedAt: chatPublications.publishedAt }) + .from(chatPublications) + .where( + and( + eq(chatPublications.companyId, companyId), + eq(chatPublications.endpointId, binding.endpointId), + eq(chatPublications.conversationId, binding.conversationId), + eq(chatPublications.state, "published"), + or( + like(chatPublications.idempotencyKey, "control:new:%"), + like(chatPublications.idempotencyKey, "control:close:%"), + ), + ), + ) + .orderBy(desc(chatPublications.publishedAt)) + .limit(1)) as Array<{ publishedAt: Date | null }>; + if (!latestControl?.publishedAt) return false; + const rows = (await dbOrTx + .select({ + action: chatActions, + receipt: agentWakeupRequests, + link: chatMessageLinks, + delivery: chatDeliveries, + comment: issueComments, + conversation: chatConversations, + endpoint: chatEndpoints, + connection: toolConnections, + }) + .from(chatActions) + .innerJoin( + agentWakeupRequests, + and( + eq(agentWakeupRequests.id, chatActions.id), + eq(agentWakeupRequests.companyId, chatActions.companyId), + ), + ) + .innerJoin( + chatMessageLinks, + and( + eq(chatMessageLinks.companyId, chatActions.companyId), + eq(chatMessageLinks.endpointId, chatActions.endpointId), + eq(chatMessageLinks.conversationId, chatActions.conversationId), + eq(chatMessageLinks.deliveryId, chatActions.deliveryId), + sql`${chatMessageLinks.commentId}::text = ${chatActions.payload}->>'commentId'`, + eq(chatMessageLinks.direction, "inbound"), + ), + ) + .innerJoin( + chatDeliveries, + and( + eq(chatDeliveries.id, chatActions.deliveryId), + eq(chatDeliveries.companyId, chatActions.companyId), + eq(chatDeliveries.endpointId, chatActions.endpointId), + eq(chatDeliveries.conversationId, chatActions.conversationId), + eq(chatDeliveries.principalId, chatActions.principalId), + ), + ) + .innerJoin( + issueComments, + and( + eq(issueComments.id, chatMessageLinks.commentId), + eq(issueComments.companyId, companyId), + eq(issueComments.issueId, issueId), + ), + ) + .innerJoin( + chatConversations, + and( + eq(chatConversations.id, binding.conversationId), + eq(chatConversations.id, chatActions.conversationId), + eq(chatConversations.companyId, companyId), + eq(chatConversations.issueId, issueId), + eq(chatConversations.endpointId, binding.endpointId), + ), + ) + .innerJoin( + chatEndpoints, + and( + eq(chatEndpoints.id, chatConversations.endpointId), + eq(chatEndpoints.companyId, companyId), + eq(chatEndpoints.assignedAgentId, run.agentId), + ), + ) + .innerJoin( + issues, + and( + eq(issues.id, issueId), + eq(issues.companyId, companyId), + eq(issues.assigneeAgentId, run.agentId), + isNull(issues.hiddenAt), + ), + ) + .innerJoin( + toolConnections, + and( + eq(toolConnections.id, chatEndpoints.connectionId), + eq(toolConnections.companyId, companyId), + ), + ) + .where( + and( + eq(chatActions.companyId, companyId), + eq(chatActions.kind, "inbound_wakeup"), + or( + eq(agentWakeupRequests.runId, run.id), + eq(agentWakeupRequests.id, owner.id), + sql`${agentWakeupRequests.payload}->>'coalescedIntoWakeupRequestId' = ${owner.id}`, + ), + isNull(issueComments.deletedAt), + sql`${issueComments.updatedAt} = ${issueComments.createdAt}`, + sql`${issueComments.createdAt} <= (select started_at from heartbeat_runs where id = ${run.id})`, + // Compare in SQL, retaining microseconds. A delayed pre-close source or + // a source preceding any later close remains internal after reopening. + notExists( + dbOrTx + .select({ id: chatPublications.id }) + .from(chatPublications) + .where( + and( + eq(chatPublications.companyId, companyId), + eq(chatPublications.endpointId, binding.endpointId), + eq(chatPublications.conversationId, binding.conversationId), + eq(chatPublications.state, "published"), + or( + like(chatPublications.idempotencyKey, "control:new:%"), + like(chatPublications.idempotencyKey, "control:close:%"), + ), + or( + isNull(chatPublications.publishedAt), + gte(chatPublications.publishedAt, chatDeliveries.receivedAt), + ), + ), + ), + ), + ), + ) + .limit(51)) as Array<{ + action: typeof chatActions.$inferSelect; + receipt: typeof agentWakeupRequests.$inferSelect; + link: typeof chatMessageLinks.$inferSelect; + delivery: typeof chatDeliveries.$inferSelect; + comment: typeof issueComments.$inferSelect; + conversation: typeof chatConversations.$inferSelect; + endpoint: typeof chatEndpoints.$inferSelect; + connection: typeof toolConnections.$inferSelect; + }>; + if ( + rows.length !== commentIds.length || + new Set(rows.map((row) => row.comment.id)).size !== commentIds.length || + rows.some((row) => !commentIds.includes(row.comment.id)) + ) + return false; + for (const row of rows) { + const { + action, + receipt, + delivery, + comment, + conversation, + endpoint, + connection, + } = row; + const payload = action.payload; + const event = parseObject(delivery.normalizedEvent); + const fence = parseObject(event.runtimeContext); + const sourceConversation = parseObject(event.conversation); + const message = parseObject(event.message); + const sentAt = + typeof message.providerSentAt === "string" && + message.providerSentAt.length <= 64 + ? Date.parse(message.providerSentAt) + : Number.NaN; + const refs = connection.credentialSecretRefs + .map((ref) => ({ + configPath: ref.configPath, + secretId: ref.secretId, + versionSelector: ref.versionSelector ?? "latest", + })) + .sort((left, right) => + `${left.configPath}:${left.secretId}:${left.versionSelector}`.localeCompare( + `${right.configPath}:${right.secretId}:${right.versionSelector}`, + ), + ); + if ( + action.status !== "processed" || + action.providerActionId !== `inbound_wakeup:${delivery.id}` || + payload.version !== 1 || + !action.principalId || + payload.issueId !== issueId || + payload.agentId !== run.agentId || + payload.commentId !== comment.id || + payload.sessionGeneration !== conversation.sessionGeneration || + delivery.state !== "processed" || + // Receipt time alone cannot make a delayed old provider message fresh. + // Missing or future provider time denies. The shared control reader below + // proves chronology against the actual command, not its later receipt. + !Number.isFinite(sentAt) || + // Historical SDK display clocks were sometimes server-local fallbacks. + // They cannot be retrospectively promoted to post-control chronology. + (endpoint.provider === "microsoft-teams" && + message.providerSentAtSource !== "teams_activity_timestamp") || + (endpoint.provider === "telegram" && + message.providerSentAtSource !== "telegram_message_date") || + sentAt > delivery.receivedAt.getTime() || + !delivery.processedAt || + !connection.enabled || + connection.status !== "active" || + fence.generation !== + Number(parseObject(endpoint.setup).runtimeGeneration ?? 0) || + fence.credentialFingerprint !== + createHash("sha256").update(JSON.stringify(refs)).digest("hex") || + sourceConversation.externalConversationId !== + conversation.externalConversationId || + sourceConversation.externalThreadId !== conversation.externalThreadId || + message.providerMessageId !== row.link.providerMessageId || + !["user", "system"].includes(String(payload.requestedByActorType)) || + typeof payload.requestedByActorId !== "string" || + receipt.payload?.issueId !== issueId || + receipt.payload?.wakeCommentId !== comment.id || + (receipt.id !== owner.id && + (receipt.status !== "coalesced" || + receipt.payload?.coalescedIntoWakeupRequestId !== owner.id)) || + (payload.requestedByActorType === "user" + ? comment.authorUserId !== payload.requestedByActorId + : comment.authorUserId !== null || + payload.requestedByActorId !== action.principalId) + ) + return false; + // The SQL above independently requires admission after every published + // control. A provider source after the command but before its confirmation + // can qualify only with complete exact proof for this published-control set. + if ( + (await readChatControlChronology( + dbOrTx, + endpoint, + { + id: conversation.externalThreadId, + channelId: conversation.externalConversationId, + }, + delivery, + { + requirePublishedControlProofForConversationId: binding.conversationId, + }, + )) !== "after_all_proven_controls" + ) + return false; + try { + assertDurableChatWakeupReceipt( + createDurableChatWakeupRequest({ + id: action.id, + companyId, + agentId: run.agentId, + issueId, + commentId: comment.id, + requestedByActorType: payload.requestedByActorType as + "user" | "system", + requestedByActorId: payload.requestedByActorId, + requestedAt: action.createdAt, + authorize: async () => {}, + }), + receipt, + ); + } catch { + return false; + } + const [lifecycle] = await dbOrTx + .select({ id: chatDeliveries.id }) + .from(chatDeliveries) + .where( + and( + eq(chatDeliveries.companyId, companyId), + eq(chatDeliveries.endpointId, binding.endpointId), + ne(chatDeliveries.state, "filtered"), + inArray(chatDeliveries.eventKind, [ + "message_updated", + "message_deleted", + "message_restored", + ]), + sql`${chatDeliveries.normalizedEvent}->'runtimeContext' = ${JSON.stringify(fence)}::jsonb`, + sql`${chatDeliveries.normalizedEvent}->'message'->>'targetProviderEventId' = ${delivery.providerEventId}`, + ), + ) + .limit(1); + if (lifecycle) return false; + } + try { + const current = await authorizeChatConversationForBoundRun( + dbOrTx, + { companyId, issueId, runId, agentId: run.agentId }, + context, + "read", + ); + return ( + current.conversationId === binding.conversationId && + current.endpointId === binding.endpointId + ); + } catch (error) { + if ( + error instanceof Error && + /^paperclip_runner_chat_attachment_(binding|destination|principal)_denied$/.test( + error.message, + ) + ) + return false; + throw error; + } +} + +export async function resolveChatOriginPublicationBindings( + dbOrTx: any, + companyId: string, + issueId: string, + runId: string | null, +): Promise { + if (!runId) return []; + const explicitEmail = await dbOrTx.select({ id: chatEndpoints.id }).from(chatEndpoints) + .innerJoin(chatConversations, eq(chatConversations.endpointId, chatEndpoints.id)) + .where(and(eq(chatConversations.companyId, companyId), eq(chatConversations.issueId, issueId), eq(chatEndpoints.publicationMode, "explicit"))).limit(1); + if (explicitEmail.length) return []; + + let originRunId = runId; + let contextSnapshot: Record | null = null; + let lineageAgentId: string | null = null; + const visitedRunIds = new Set(); + // Every hop needs its own durable authorization receipt. A task may continue + // many times; an arbitrary hop limit must not silently discard its replies. + // The visited set rejects cycles while keeping valid long lineages readable. + while (true) { + if (visitedRunIds.has(originRunId)) return []; + visitedRunIds.add(originRunId); + const run = await dbOrTx + .select({ + agentId: heartbeatRuns.agentId, + responsibleUserId: heartbeatRuns.responsibleUserId, + contextSnapshot: sql | null>` + case when jsonb_typeof(${heartbeatRuns.contextSnapshot}) = 'object' then + (select coalesce(jsonb_object_agg(entry.key, entry.value), '{}'::jsonb) + from jsonb_each(${heartbeatRuns.contextSnapshot}) as entry + where entry.key in ('issueId', 'taskId', 'wakeCommentIds', 'wakeCommentId', + 'commentId', 'source', 'sourceRunId', 'interactionId', 'endpointId')) + else ${heartbeatRuns.contextSnapshot} end`, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, originRunId), + eq(heartbeatRuns.companyId, companyId), + ), + ) + .then( + ( + rows: Array<{ + agentId: string; + responsibleUserId: string | null; + contextSnapshot: Record | null; + }>, + ) => rows[0] ?? null, + ); + if (!run?.agentId) return []; + if (lineageAgentId === null) lineageAgentId = run.agentId; + else if (run.agentId !== lineageAgentId) return []; + const snapshot = run?.contextSnapshot; + if (!snapshot) return []; + const snapshotIssueId = + readStringFromRecord(snapshot, "issueId") ?? + readStringFromRecord(snapshot, "taskId"); + if (snapshotIssueId && snapshotIssueId !== issueId) return []; + + const boardBindings = await slackBoardReplyBindings(dbOrTx, { + companyId, issueId, agentId: lineageAgentId!, + commentIds: readChatWakeCommentIds(snapshot), userId: run.responsibleUserId, + }); + if (boardBindings.length) return boardBindings; + + // A native runner can emit its continuation immediately after the durable + // `request.resolve` command is queued, before the delivery worker records + // its terminal receipt. The source run plus the already-published native + // prompt is a stronger causal edge than that tiny receipt-commit window. + const inFlightNativeInteractions: Array<{ interactionId: string }> = + await dbOrTx + .select({ + interactionId: issueQuestionResponseDeliveries.interactionId, + }) + .from(issueQuestionResponseDeliveries) + .innerJoin( + issueThreadInteractions, + and( + eq( + issueThreadInteractions.companyId, + issueQuestionResponseDeliveries.companyId, + ), + eq( + issueThreadInteractions.issueId, + issueQuestionResponseDeliveries.issueId, + ), + eq( + issueThreadInteractions.id, + issueQuestionResponseDeliveries.interactionId, + ), + ), + ) + .where( + and( + eq(issueQuestionResponseDeliveries.companyId, companyId), + eq(issueQuestionResponseDeliveries.issueId, issueId), + eq(issueQuestionResponseDeliveries.sourceRunId, originRunId), + eq(issueQuestionResponseDeliveries.status, "delivering"), + eq(issueThreadInteractions.kind, "ask_user_questions"), + eq(issueThreadInteractions.status, "answered"), + ), + ); + for (const candidate of inFlightNativeInteractions) { + const promptBindings = await resolvePublishedInteractionPromptBindings( + dbOrTx, + companyId, + issueId, + candidate.interactionId, + lineageAgentId!, + ); + if (promptBindings.length > 0) return promptBindings; + } + + // A question response can be delivered directly into an already-running + // successor without rewriting that run's pre-existing context snapshot. + // The durable delivery receipt is the authoritative causal edge and must + // win even when the target run already has a different `chat:*` source; + // otherwise a Slack answer steered into a Telegram-origin run on the same + // issue could publish the result to the wrong provider identity. + const steeredQuestionParents: Array<{ + interactionId: string; + sourceRunId: string | null; + }> = await dbOrTx + .select({ + interactionId: issueQuestionResponseDeliveries.interactionId, + sourceRunId: issueQuestionResponseDeliveries.sourceRunId, + }) + .from(issueQuestionResponseDeliveries) + .innerJoin( + issueThreadInteractions, + and( + eq( + issueThreadInteractions.companyId, + issueQuestionResponseDeliveries.companyId, + ), + eq( + issueThreadInteractions.issueId, + issueQuestionResponseDeliveries.issueId, + ), + eq( + issueThreadInteractions.id, + issueQuestionResponseDeliveries.interactionId, + ), + eq( + issueThreadInteractions.sourceRunId, + issueQuestionResponseDeliveries.sourceRunId, + ), + ), + ) + .where( + and( + eq(issueQuestionResponseDeliveries.companyId, companyId), + eq(issueQuestionResponseDeliveries.issueId, issueId), + eq(issueQuestionResponseDeliveries.targetRunId, originRunId), + inArray(issueQuestionResponseDeliveries.status, [ + "delivered", + "fallback_queued", + ]), + eq(issueThreadInteractions.kind, "ask_user_questions"), + eq(issueThreadInteractions.status, "answered"), + ), + ); + if (steeredQuestionParents.length > 0) { + const parentRunIds = [ + ...new Set( + steeredQuestionParents + .map((candidate) => candidate.sourceRunId) + .filter((candidate): candidate is string => Boolean(candidate)), + ), + ]; + // Multiple responses may coalesce into one target only when they share + // the same verified origin. Incompatible origins are intentionally + // internal rather than guessing a provider destination. + if (parentRunIds.length !== 1 || !isUuidLike(parentRunIds[0])) return []; + const promptBindings = ( + await Promise.all( + [ + ...new Set( + steeredQuestionParents.map( + (candidate) => candidate.interactionId, + ), + ), + ].map((interactionId) => + resolvePublishedInteractionPromptBindings( + dbOrTx, + companyId, + issueId, + interactionId, + lineageAgentId!, + ), + ), + ) + ).flat(); + if (promptBindings.length > 0) { + return [ + ...new Map( + promptBindings.map((binding) => [binding.conversationId, binding]), + ).values(), + ]; + } + // Native delivery resumes the source run in place and records that same + // run as its target. That is already the chat root, not a lineage hop. + if (parentRunIds[0] !== originRunId) { + originRunId = parentRunIds[0]; + continue; + } + } + + const contextSource = readStringFromRecord(snapshot, "source"); + if (contextSource === "tool_action_review") { + // Review results use their own durable wake, not the ordinary interaction + // response key. Attest the entire batch before following its origin; the + // model's context/sourceRunId alone never grants publication authority. + const [wake] = await dbOrTx.select({ payload: agentWakeupRequests.payload, + idempotencyKey: agentWakeupRequests.idempotencyKey }) + .from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.agentId, lineageAgentId!), + eq(agentWakeupRequests.runId, originRunId), + like(agentWakeupRequests.idempotencyKey, "tool-action-response:%"), + )).limit(1); + const requestIds = wake?.payload?.toolActionRequestIds; + const sourceRunId = readStringFromRecord(snapshot, "sourceRunId"); + if (!Array.isArray(requestIds) || requestIds.length === 0 || + requestIds.some((id: unknown) => typeof id !== "string" || !isUuidLike(id)) || + !sourceRunId || !isUuidLike(sourceRunId) || sourceRunId !== wake.payload.sourceRunId || + wake.idempotencyKey !== `tool-action-response:${requestIds[0]}` || + snapshot.interactionId !== wake.payload.interactionId) return []; + const receipts = await dbOrTx.select({ id: toolActionRequests.id }) + .from(toolActionRequests) + .innerJoin(toolInvocations, and( + eq(toolInvocations.id, toolActionRequests.invocationId), + eq(toolInvocations.companyId, companyId), + eq(toolInvocations.issueId, issueId), + eq(toolInvocations.agentId, lineageAgentId!), + eq(toolInvocations.runId, sourceRunId), + )) + .innerJoin(toolActionDeliveries, and( + eq(toolActionDeliveries.actionRequestId, toolActionRequests.id), + eq(toolActionDeliveries.companyId, companyId), + eq(toolActionDeliveries.issueId, issueId), + eq(toolActionDeliveries.interactionId, toolActionRequests.interactionId), + )) + .where(and( + eq(toolActionRequests.companyId, companyId), + eq(toolActionRequests.issueId, issueId), + eq(toolActionRequests.requestedByAgentId, lineageAgentId!), + inArray(toolActionRequests.id, requestIds), + inArray(toolActionRequests.status, ["executed", "failed", "rejected", "expired", "cancelled"]), + )); + if (receipts.length !== requestIds.length) return []; + originRunId = sourceRunId; + continue; + } + if (contextSource?.startsWith("chat:")) { + contextSnapshot = snapshot; + break; + } + if (contextSource === "connection_intent.resolved") { + const interactionId = readStringFromRecord(snapshot, "interactionId"); + if (!interactionId || !isUuidLike(interactionId)) return []; + const sourceRunId = await resolveConnectionIntentContinuationSource(dbOrTx, { + companyId, issueId, interactionId, runId: originRunId, agentId: lineageAgentId!, + }); + if (!sourceRunId) return []; + // An automation can request access on an already-linked Slack task. + // Its validated receipt and published prompt preserve that conversation. + const promptBindings = await resolvePublishedInteractionPromptBindings( + dbOrTx, companyId, issueId, interactionId, lineageAgentId!, + ); + if (promptBindings.length > 0) return promptBindings; + originRunId = sourceRunId; + continue; + } + if (contextSource === "issue.monitor") { + const sourceRunId = await readNativeMonitorSource(dbOrTx, { companyId, issueId, runId: originRunId }); + if (!sourceRunId) return []; + originRunId = sourceRunId; + continue; + } + if (contextSource === "native_status_decision") { + const sourceRunId = await readNativeContinuationSource(dbOrTx, { companyId, issueId, runId: originRunId }); + if (!sourceRunId) return []; + originRunId = sourceRunId; + continue; + } + if ( + contextSource !== "issue.interaction.respond" && + contextSource !== "issue.interaction.accept" && + contextSource !== "issue.interaction.reject" && + contextSource !== "issue.interaction.verdicts" && + contextSource !== "issue.interaction.cancel" && + contextSource !== "issue.interaction.withdraw" && + contextSource !== "external_chat.interaction.resolve" + ) + return []; + + const interactionId = readStringFromRecord(snapshot, "interactionId"); + const sourceRunId = readStringFromRecord(snapshot, "sourceRunId"); + if ( + !interactionId || + !isUuidLike(interactionId) || + !sourceRunId || + !isUuidLike(sourceRunId) + ) + return []; + const interaction = await dbOrTx + .select({ + kind: issueThreadInteractions.kind, + status: issueThreadInteractions.status, + result: issueThreadInteractions.result, + sourceRunId: issueThreadInteractions.sourceRunId, + }) + .from(issueThreadInteractions) + .where( + and( + eq(issueThreadInteractions.id, interactionId), + eq(issueThreadInteractions.companyId, companyId), + eq(issueThreadInteractions.issueId, issueId), + eq(issueThreadInteractions.sourceRunId, sourceRunId), + ), + ) + .then( + ( + rows: Array<{ + kind: string; + status: string; + result: unknown; + sourceRunId: string | null; + }>, + ) => rows[0] ?? null, + ); + if (!interaction?.sourceRunId) return []; + const isAnsweredQuestion = + interaction.kind === "ask_user_questions" && + interaction.status === "answered"; + const isResolvedConfirmation = + ["request_confirmation", "request_checkbox_confirmation", "suggest_tasks"].includes(interaction.kind) && + (interaction.status === "accepted" || interaction.status === "rejected"); + const isCancelledSupportedInteraction = + interaction.status === "cancelled" && + ["ask_user_questions", "request_confirmation", "request_checkbox_confirmation", "request_item_verdicts", "suggest_tasks"].includes(interaction.kind); + const isItemVerdictResponse = interaction.kind === "request_item_verdicts" && + (interaction.status === "answered" || interaction.status === "pending") && + contextSource === "issue.interaction.verdicts"; + if ( + !isItemVerdictResponse && + !isAnsweredQuestion && + !isResolvedConfirmation && + !isCancelledSupportedInteraction + ) + return []; + const responseDelivery = isAnsweredQuestion + ? await dbOrTx + .select({ + status: issueQuestionResponseDeliveries.status, + targetRunId: issueQuestionResponseDeliveries.targetRunId, + }) + .from(issueQuestionResponseDeliveries) + .where( + and( + eq(issueQuestionResponseDeliveries.companyId, companyId), + eq(issueQuestionResponseDeliveries.issueId, issueId), + eq(issueQuestionResponseDeliveries.interactionId, interactionId), + eq(issueQuestionResponseDeliveries.sourceRunId, sourceRunId), + ), + ) + .then( + (rows: Array<{ status: string; targetRunId: string | null }>) => + rows[0] ?? null, + ) + : null; + const recordedTarget = + isAnsweredQuestion && + responseDelivery && + (responseDelivery.status === "delivered" || + responseDelivery.status === "fallback_queued") + ? responseDelivery.targetRunId + : null; + if (recordedTarget && recordedTarget !== originRunId) return []; + const continuationIdempotencyKey = isAnsweredQuestion + ? `question-response:${interactionId}` + : `interaction:${interactionId}:${interaction.status}`; + const itemVerdictWake = isItemVerdictResponse + ? await resolveItemVerdictContinuationWake(dbOrTx, { + companyId, issueId, interactionId, sourceRunId, runId: originRunId, + agentId: lineageAgentId!, result: interaction.result, + }) + : null; + const durableWakeTarget = isItemVerdictResponse ? itemVerdictWake : recordedTarget + ? null + : await dbOrTx + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + eq( + agentWakeupRequests.idempotencyKey, + continuationIdempotencyKey, + ), + eq(agentWakeupRequests.agentId, lineageAgentId!), + eq(agentWakeupRequests.runId, originRunId), + inArray(agentWakeupRequests.status, [ + "queued", + "claimed", + "running", + "succeeded", + "completed", + "failed", + "cancelled", + "coalesced", + "deferred_issue_execution", + "retrying", + "scheduled_retry", + ]), + ), + ) + .then((rows: Array<{ id: string }>) => rows[0] ?? null); + if (recordedTarget !== originRunId && !durableWakeTarget) return []; + const promptBindings = await resolvePublishedInteractionPromptBindings( + dbOrTx, + companyId, + issueId, + interactionId, + lineageAgentId!, + ); + if (promptBindings.length > 0) return promptBindings; + originRunId = interaction.sourceRunId; + } + if (!contextSnapshot || !lineageAgentId) return []; + + const wakeCommentIds = readChatWakeCommentIds(contextSnapshot); + if (wakeCommentIds.length === 0) return []; + const originEndpointId = readStringFromRecord(contextSnapshot, "endpointId"); + + const bindings: Array = + await dbOrTx + .select({ + companyId: chatConversations.companyId, + conversationId: chatConversations.id, + endpointId: chatConversations.endpointId, + hasControl: sql`exists (select 1 from chat_publications control where control.company_id = ${chatConversations.companyId} and control.endpoint_id = ${chatConversations.endpointId} and control.conversation_id = ${chatConversations.id} and control.state = 'published' and (control.idempotency_key like 'control:new:%' or control.idempotency_key like 'control:close:%'))`, + }) + .from(chatMessageLinks) + .innerJoin( + chatConversations, + and( + eq(chatConversations.companyId, chatMessageLinks.companyId), + eq(chatConversations.endpointId, chatMessageLinks.endpointId), + eq(chatConversations.id, chatMessageLinks.conversationId), + ), + ) + .innerJoin( + chatEndpoints, + and( + eq(chatEndpoints.companyId, chatConversations.companyId), + eq(chatEndpoints.id, chatConversations.endpointId), + ), + ) + .where( + and( + eq(chatMessageLinks.companyId, companyId), + eq(chatMessageLinks.direction, "inbound"), + inArray(chatMessageLinks.commentId, wakeCommentIds), + eq(chatConversations.issueId, issueId), + eq(chatEndpoints.assignedAgentId, lineageAgentId), + ...(originEndpointId + ? [eq(chatConversations.endpointId, originEndpointId)] + : []), + ), + ); + const authorized: ChatPublicationBinding[] = []; + // A coalesced batch has one inbound link per comment. Prove each destination + // once, not once per link (the proof itself still checks the complete batch). + const candidates = new Map( + bindings.map((binding) => [binding.conversationId, binding]), + ); + for (const { hasControl, ...binding } of candidates.values()) { + if ( + !hasControl || + (await freshChatSourceAfterPublishedControl( + dbOrTx, + companyId, + issueId, + originRunId, + binding, + )) + ) + authorized.push(binding); + } + return authorized; +} + +async function resolveCommentResponsibleUserId( + dbOrTx: any, + companyId: string, + createdByRunId: string | null, + actorResponsibleUserId: string | null | undefined, +): Promise { + const actorValue = actorResponsibleUserId?.trim() || null; + if (actorValue) return actorValue; + if (!createdByRunId) return null; + return dbOrTx + .select({ responsibleUserId: heartbeatRuns.responsibleUserId }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, createdByRunId), + eq(heartbeatRuns.companyId, companyId), + ), + ) + .then( + (rows: Array<{ responsibleUserId: string | null }>) => + rows[0]?.responsibleUserId?.trim() || null, + ); +} + +function withAgentCommentAuthorizationMetadata( + metadata: IssueCommentMetadata | null, + authorizationReason: string | null | undefined, +): IssueCommentMetadata { + const reason = authorizationReason?.trim() || "internal_agent_write"; + return { + version: 1, + ...(metadata?.sourceRunId !== undefined + ? { sourceRunId: metadata.sourceRunId } + : {}), + authorizationReason: reason, + sections: metadata?.sections.length + ? metadata.sections + : [ + { + title: "Authorization", + rows: [{ type: "key_value", label: "Reason", value: reason }], + }, + ], + }; +} + +/** + * Chat-origin runs may create internal presentation and bookkeeping comments. + * Only comments produced through an explicit agent write surface are authored + * for the external conversation and therefore eligible for auto-publication. + */ +export function isExplicitExternalAgentComment( + metadata: IssueCommentMetadata | null | undefined, +): boolean { + const reason = metadata?.authorizationReason?.trim() ?? ""; + return reason === "paperclip_runner_protocol" || reason.startsWith("allow_"); +} + +type SelectedChatPresentationAttachment = { + id: string; + commentId: string; + commentMetadata: IssueCommentMetadata | null; + originalFilename: string | null; +}; + +async function listSelectedChatPresentationAttachments( + dbOrTx: any, + input: { + companyId: string; + issueId: string; + agentId: string; + runId: string; + }, +): Promise { + const rows = await dbOrTx + .select({ + id: issueAttachments.id, + commentId: issueComments.id, + commentMetadata: issueComments.metadata, + originalFilename: assets.originalFilename, + }) + .from(issueAttachments) + .innerJoin(assets, eq(issueAttachments.assetId, assets.id)) + .innerJoin( + issueComments, + eq(issueAttachments.issueCommentId, issueComments.id), + ) + .where( + and( + eq(issueAttachments.companyId, input.companyId), + eq(issueAttachments.issueId, input.issueId), + eq(issueAttachments.originatingRunId, input.runId), + eq(assets.companyId, input.companyId), + eq(assets.createdByAgentId, input.agentId), + eq(issueComments.companyId, input.companyId), + eq(issueComments.issueId, input.issueId), + eq(issueComments.createdByRunId, input.runId), + eq(issueComments.authorType, "agent"), + eq(issueComments.authorAgentId, input.agentId), + isNull(issueComments.deletedAt), + sql`( + coalesce(${issueComments.metadata}->>'authorizationReason', '') = 'paperclip_runner_protocol' + or left(coalesce(${issueComments.metadata}->>'authorizationReason', ''), 6) = 'allow_' + )`, + ), + ) + .orderBy( + asc(issueComments.createdAt), + asc(issueComments.id), + asc(issueAttachments.createdAt), + asc(issueAttachments.id), + ); + return (rows as SelectedChatPresentationAttachment[]).filter((row) => + isExplicitExternalAgentComment(row.commentMetadata), + ); +} + +/** + * Best-effort agent attribution for comments whose stored author is a non-human + * sentinel (e.g. `local-board`). Callers MUST pre-filter `comments` to drop any + * comment whose `authorUserId` maps to a genuine user profile so a real board / + * user comment is never reattributed. + * + * Only LOSSLESS signals are used — a comment is reattributed solely when a run + * provably authored it. Pure run-window timing overlap is intentionally NOT a + * signal: because agents post through the `local-board` subprocess, an agent + * comment and a genuine human board comment are indistinguishable rows, so any + * timing-based guess mis-attributes human board comments that merely coincided + * with an agent run (Option A). + * + * Tiers, in descending confidence (first match wins per comment): + * 1. `run_id` — the comment's own `createdByRunId` resolves to an agent run + * (lossless: that run authored the comment). + * 2. `run_log_comment_post` — an overlapping run log contains the explicit + * `comment id: {id}` post marker (lossless: the run recorded posting it). + */ +export function deriveIssueCommentRunLogAttribution( + comments: readonly IssueCommentRunLogAttributionCandidate[], + runs: readonly IssueCommentRunLogAttributionRun[], +) { + const derivedByCommentId = new Map(); + const runById = new Map(runs.map((run) => [run.runId, run] as const)); + + for (const comment of comments) { + if (comment.authorAgentId || !comment.authorUserId) continue; + + // Tier 1: the comment carries the run that authored it. Lossless even when + // the author was recorded as the `local-board` sentinel. + if (comment.createdByRunId) { + const ownRun = runById.get(comment.createdByRunId); + if (ownRun?.agentId) { + derivedByCommentId.set(comment.id, { + derivedAuthorAgentId: ownRun.agentId, + derivedCreatedByRunId: ownRun.runId, + derivedAuthorSource: "run_id", + }); + continue; + } + } + + const commentCreatedAtMs = toTimestampMs(comment.createdAt); + if (commentCreatedAtMs === null) continue; + + const overlappingRuns: Array<{ + run: IssueCommentRunLogAttributionRun; + runEndMs: number; + }> = []; + for (const run of runs) { + const runStartMs = toTimestampMs(run.startedAt ?? run.createdAt); + const runEndMs = toTimestampMs(run.finishedAt ?? run.createdAt); + if (runStartMs === null || runEndMs === null) continue; + if ( + commentCreatedAtMs < runStartMs || + commentCreatedAtMs > + runEndMs + ISSUE_COMMENT_RUN_LOG_DERIVATION_END_SLACK_MS + ) { + continue; + } + overlappingRuns.push({ run, runEndMs }); + } + + // Tier 2: an overlapping run log explicitly recorded posting this comment. + let bestLogMatch: { + runId: string; + agentId: string; + distanceMs: number; + } | null = null; + for (const { run, runEndMs } of overlappingRuns) { + if (!run.logContent.includes(`comment id: ${comment.id}`)) continue; + const distanceMs = Math.abs(runEndMs - commentCreatedAtMs); + if (!bestLogMatch || distanceMs < bestLogMatch.distanceMs) { + bestLogMatch = { runId: run.runId, agentId: run.agentId, distanceMs }; + } + } + if (bestLogMatch) { + derivedByCommentId.set(comment.id, { + derivedAuthorAgentId: bestLogMatch.agentId, + derivedCreatedByRunId: bestLogMatch.runId, + derivedAuthorSource: "run_log_comment_post", + }); + continue; + } + + // No lossless signal — leave unresolved. A pure run-window timing overlap is + // deliberately NOT enough to reattribute (it cannot tell an agent comment + // from a human board comment that happened during the run). + } + + return derivedByCommentId; +} + +// Express's default `qs` parser binds repeated query keys to a `string[]`, +// so a request like `?status=todo&status=in_progress` arrives here as an +// array. Single-key + comma-separated forms remain valid too; normalize the +// supported shapes once so the service contract matches runtime reality. +export function parseStatusFilter( + input: string | readonly string[] | undefined, +): string[] { + if (input === undefined || input === null) return []; + const entries = Array.isArray(input) + ? input + : typeof input === "string" + ? [input] + : []; + return entries + .flatMap((entry) => (typeof entry === "string" ? entry.split(",") : [])) + .map((status) => status.trim()) + .filter(Boolean); +} + +export interface IssueFilters { + attention?: "blocked"; + status?: string | readonly string[]; + /** + * Filter by assignee agent ID. + * - `string` (UUID): match issues assigned to that agent. + * - `null`: match unassigned issues (IS NULL). + * - The literal string `"null"` is also accepted as a sentinel for `null` + * so that query-string callers can pass `?assigneeAgentId=null` directly. + * The route layer normalises it before calling the service, but the service + * also normalises it for direct callers. + */ + assigneeAgentId?: string | null; + participantAgentId?: string; + issueIds?: string[]; + listProjection?: "index" | "rows"; + parentIds?: string[]; + oneChildPerParent?: boolean; + assigneeUserId?: string; + touchedByUserId?: string; + inboxArchivedByUserId?: string; + unreadForUserId?: string; + projectId?: string; + workspaceId?: string; + executionWorkspaceId?: string; + parentId?: string; + descendantOf?: string; + createdFromIssueId?: string; + labelId?: string; + originKind?: string; + originKindPrefix?: string; + originId?: string; + includeAgentCreatedTasks?: boolean; + /** Authenticated inbox discovery can retain existing participation. */ + creationVisibilityUserId?: string; + includeRoutineExecutions?: boolean; + excludeRoutineExecutions?: boolean; + includePluginOperations?: boolean; + includeBlockedBy?: boolean; + includeBlockedInboxAttention?: boolean; + includeLiveDescendantSummary?: boolean; + hasPlanDocument?: boolean; + lowTrustBoundary?: LowTrustBoundary & { companyId: string }; + readCondition?: SQL; + q?: string; + limit?: number; + offset?: number; + sortField?: "updated" | "id"; + afterId?: string; + sortDir?: "asc" | "desc"; + /** ISO 8601 timestamp — only return issues with updatedAt strictly after this value. */ + updatedSince?: string; +} + +type IssueRow = typeof issues.$inferSelect & { externalConversationState?: "active" | "waiting" | null }; +type IssueLabelRow = typeof labels.$inferSelect; +type IssuePlanDecompositionRow = typeof issuePlanDecompositions.$inferSelect; +type IssueActiveRunRow = { + execution?: ExecutionProjection; + id: string; + status: string; + agentId: string; + invocationSource: string; + triggerDetail: string | null; + startedAt: Date | null; + finishedAt: Date | null; + createdAt: Date; +}; +type IssueScheduledRetryRow = { + runId: string; + status: "scheduled_retry" | "queued" | "running" | "cancelled"; + agentId: string; + agentName: string | null; + retryOfRunId: string | null; + scheduledRetryAt: Date | null; + scheduledRetryAttempt: number; + scheduledRetryReason: string | null; + retryExhaustedReason?: string | null; + error?: string | null; + errorCode?: string | null; +}; +type IssueWithLabels = IssueRow & { + labels: IssueLabelRow[]; + labelIds: string[]; + watchdog?: IssueWatchdogSummary | null; +}; +type IssueWithLabelsAndRun = IssueWithLabels & { + activeRun: IssueActiveRunRow | null; +}; +type IssueUserCommentStats = { + issueId: string; + isUnreadForMe: boolean; + myLastCommentAt: Date | null; + lastExternalCommentAt: Date | null; +}; +type IssueReadStat = { + issueId: string; + myLastReadAt: Date | null; +}; +type IssueLastActivityStat = { + issueId: string; + latestCommentAt: Date | null; + latestLogAt: Date | null; +}; + +function serializeAcceptedPlanDecomposition( + decomposition: IssuePlanDecompositionRow, +): AcceptedPlanDecomposition { + return { + id: decomposition.id, + companyId: decomposition.companyId, + sourceIssueId: decomposition.sourceIssueId, + acceptedPlanRevisionId: decomposition.acceptedPlanRevisionId, + acceptedInteractionId: decomposition.acceptedInteractionId, + status: decomposition.status as AcceptedPlanDecomposition["status"], + requestFingerprint: decomposition.requestFingerprint, + // Intentionally omit requestedChildren here; the API only needs stable counts + // and child ids, while the durable table keeps the full child draft payload. + requestedChildCount: decomposition.requestedChildCount, + childIssueIds: normalizeIssuePlanDecompositionChildIds( + decomposition.childIssueIds, + ), + ownerAgentId: decomposition.ownerAgentId, + ownerUserId: decomposition.ownerUserId, + ownerRunId: decomposition.ownerRunId, + completedAt: decomposition.completedAt, + createdAt: decomposition.createdAt, + updatedAt: decomposition.updatedAt, + }; +} +type IssueUserContextInput = { + createdByAgentId?: string | null; + originKind?: string | null; + createdByUserId: string | null; + assigneeUserId: string | null; + createdAt: Date | string; + updatedAt: Date | string; +}; +type ProjectGoalReader = Pick; +type DbReader = Pick; +/** Conversation containers cannot acquire new child edges, even with the experiment disabled. */ +async function assertExecutionTaskParent(db: Db, companyId: string, parentId?: string | null) { + if (!parentId) return; + const [parent] = await db.select({ conversationAgentId: issues.conversationAgentId }) + .from(issues).where(and(eq(issues.id, parentId), eq(issues.companyId, companyId))); + if (parent?.conversationAgentId) throw unprocessable("Conversations cannot have new subtasks; create a task in a project instead"); +} + +type DbTransaction = Parameters[0]>[0]; +type IssueCreateInput = Omit & { + workspaceSelection?: TaskWorkspaceSelection; + workspaceSelectionSource?: "explicit" | "channel"; + workspaceSelectionActor?: AuthorizationActor; + title?: string; + initialPlan?: string | null; + /** Internal, server-derived provenance for an accepted in-app chat suggestion. */ + conversationHandoff?: { conversationId: string; sessionGeneration: number }; + labelIds?: string[]; + blockedByIssueIds?: string[]; + inheritExecutionWorkspaceFromIssueId?: string | null; + skipExecutionWorkspaceInheritance?: boolean; + watchdog?: { agentId: string; instructions?: string | null } | null; + watchdogActorRunId?: string | null; + actorRunId?: string | null; + actorResponsibleUserId?: string | null; + trustExplicitResponsibleUserId?: boolean; + idempotencyKey?: string | null; + allowDuplicate?: boolean; + assertCanReuseIssue?: (issue: typeof issues.$inferSelect) => Promise; + onDeduplicated?: (reason: "idempotency_key" | "recent_open_title") => void; +}; +type IssueChildCreateInput = IssueCreateInput & { + acceptanceCriteria?: string[]; + blockParentUntilDone?: boolean; + executionWorkspaceInheritanceMode?: "linkage" | "strategy_only"; + actorAgentId?: string | null; + actorUserId?: string | null; +}; +type AcceptedPlanDecompositionInput = { + acceptedPlanRevisionId: string; + children: IssueChildCreateInput[]; + actorAgentId?: string | null; + actorUserId?: string | null; + actorRunId?: string | null; +}; +type AcceptedPlanDocumentInteraction = { + id: string; +}; +type IssueRelationSummaryMap = { + blockedBy: IssueRelationIssueSummary[]; + blocks: IssueRelationIssueSummary[]; +}; +type IssueBlockerDiagnosticsIssueRow = { + id: string; + companyId: string; + projectId: string | null; + parentId: string | null; + identifier: string | null; + title: string; + status: (typeof ALL_ISSUE_STATUSES)[number]; + priority: string; + assigneeAgentId: string | null; + assigneeUserId: string | null; +}; +type IssueWakeDiagnosticsWakeRequestRow = { + agentId: string; + source: string; + reason: string | null; + status: string; + coalescedCount: number; + runId: string | null; + requestedAt: Date; + claimedAt: Date | null; + finishedAt: Date | null; + error: string | null; +}; +type IssueWakeDiagnosticsActivityRow = { + action: string; + entityType: string; + entityId: string; + agentId: string | null; + runId: string | null; + details: Record | null; + createdAt: Date; +}; +type IssueSubtreeDiagnosticsIssueRow = IssueBlockerDiagnosticsIssueRow & { + depth: number; + createdAt: Date; + updatedAt: Date; +}; +type IssueSubtreeDiagnosticsBlockerRow = IssueBlockerDiagnosticsIssueRow & { + blockedIssueId: string; + relationCreatedAt: Date; +}; +type IssueSubtreeDiagnosticsWakeRequestRow = + IssueWakeDiagnosticsWakeRequestRow & { + issueId: string; + }; +type IssueSubtreeDiagnosticsActivityRow = IssueWakeDiagnosticsActivityRow & { + issueId: string; +}; +type IssueSubtreeDiagnosticsBlockerResultRow = + IssueSubtreeDiagnosticsBlockerRow & { + rowNumber: number | string; + }; +type IssueSubtreeDiagnosticsWakeRequestResultRow = + IssueSubtreeDiagnosticsWakeRequestRow & { + rowNumber: number | string; + }; +type IssueSubtreeDiagnosticsActivityResultRow = + IssueSubtreeDiagnosticsActivityRow & { + rowNumber: number | string; + }; +export type IssueDependencyReadiness = { + issueId: string; + blockerIssueIds: string[]; + unresolvedBlockerIssueIds: string[]; + unresolvedBlockerCount: number; + /** Blockers whose status is `done` but whose execution workspace has not yet finalized. */ + pendingFinalizeBlockerIssueIds: string[]; + allBlockersDone: boolean; + isDependencyReady: boolean; +}; +export type ChildIssueCompletionSummary = { + id: string; + identifier: string | null; + title: string; + status: string; + priority: string; + assigneeAgentId: string | null; + assigneeUserId: string | null; + updatedAt: Date; + summary: string | null; +}; + +function sameRunLock(checkoutRunId: string | null, actorRunId: string | null) { + if (actorRunId) return checkoutRunId === actorRunId; + return checkoutRunId == null; +} + +export const TERMINAL_HEARTBEAT_RUN_STATUSES = new Set([ + "succeeded", + "interrupted", + "failed", + "cancelled", + "timed_out", +]); +const ISSUE_LIST_DESCRIPTION_MAX_CHARS = 1200; +const ISSUE_LIST_DESCRIPTION_MAX_BYTES = ISSUE_LIST_DESCRIPTION_MAX_CHARS * 4; + +function escapeLikePattern(value: string): string { + return value.replace(/[\\%_]/g, "\\$&"); +} + +export function clampIssueListLimit(limit: number): number { + return Math.min(ISSUE_LIST_MAX_LIMIT, Math.max(1, Math.floor(limit))); +} + +function chunkList(values: T[], size: number): T[][] { + const chunks: T[][] = []; + for (let index = 0; index < values.length; index += size) { + chunks.push(values.slice(index, index + size)); + } + return chunks; +} + +function truncateInlineSummary( + value: string | null | undefined, + maxChars = CHILD_COMPLETION_SUMMARY_BODY_MAX_CHARS, +) { + const normalized = value?.trim(); + if (!normalized) return null; + return normalized.length > maxChars + ? `${normalized.slice(0, Math.max(0, maxChars - 15)).trimEnd()} [truncated]` + : normalized; +} + +function truncateByCodePoint(value: string, maxChars: number): string { + if (value.length <= maxChars) return value; + return Array.from(value).slice(0, maxChars).join(""); +} + +function decodeDatabaseTextPreview( + value: string | null | undefined, + maxChars: number, +): string | null { + if (value == null) return null; + return truncateByCodePoint( + Buffer.from(value, "base64").toString("utf8"), + maxChars, + ); +} + +function appendAcceptanceCriteriaToDescription( + description: string | null | undefined, + acceptanceCriteria: string[] | undefined, +) { + const criteria = (acceptanceCriteria ?? []) + .map((item) => item.trim()) + .filter(Boolean); + if (criteria.length === 0) return description ?? null; + const base = description?.trim() ?? ""; + const criteriaMarkdown = [ + "## Acceptance Criteria", + "", + ...criteria.map((item) => `- ${item}`), + ].join("\n"); + return base ? `${base}\n\n${criteriaMarkdown}` : criteriaMarkdown; +} + +function normalizeAcceptedPlanDecompositionFingerprintValue( + value: unknown, +): unknown { + if (value === undefined) return null; + if ( + value == null || + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ) { + return value; + } + if (value instanceof Date) return value.toISOString(); + if (Array.isArray(value)) { + return value.map((item) => + normalizeAcceptedPlanDecompositionFingerprintValue(item), + ); + } + if (typeof value === "object") { + const record = value as Record; + return Object.fromEntries( + Object.keys(record) + .sort() + .map((key) => [ + key, + normalizeAcceptedPlanDecompositionFingerprintValue(record[key]), + ]), + ); + } + return String(value); +} + +const ACCEPTED_PLAN_DECOMPOSITION_FINGERPRINT_CHILD_METADATA_KEYS = new Set([ + "id", + "companyId", + "parentId", + "identifier", + "checkoutRunId", + "executionRunId", + "executionLockedAt", + "startedAt", + "completedAt", + "cancelledAt", + "hiddenAt", + "createdAt", + "updatedAt", + "createdByAgentId", + "createdByUserId", + "updatedByAgentId", + "updatedByUserId", + "actorAgentId", + "actorUserId", + "executionWorkspaceInheritanceMode", + "skipExecutionWorkspaceInheritance", +]); + +function normalizeAcceptedPlanDecompositionFingerprintChild( + child: IssueChildCreateInput, +) { + return Object.fromEntries( + Object.entries(child).filter( + ([key]) => + !ACCEPTED_PLAN_DECOMPOSITION_FINGERPRINT_CHILD_METADATA_KEYS.has(key), + ), + ); +} + +function createAcceptedPlanDecompositionRequestFingerprint(input: { + acceptedPlanRevisionId: string; + children: IssueChildCreateInput[]; +}) { + const canonical = JSON.stringify( + normalizeAcceptedPlanDecompositionFingerprintValue({ + acceptedPlanRevisionId: input.acceptedPlanRevisionId, + children: input.children.map( + normalizeAcceptedPlanDecompositionFingerprintChild, + ), + }), + ); + return createHash("sha256").update(canonical).digest("hex"); +} + +function normalizeIssuePlanDecompositionChildIds(value: unknown): string[] { + if (!Array.isArray(value)) return []; + return value.filter( + (item): item is string => typeof item === "string" && item.length > 0, + ); +} + +export function readAcceptedPlanConfirmationTarget( + payload: unknown, + fallbackIssueId?: string, +): { + revisionId: string; + key: string; + issueId: string; +} | null { + if (!payload || typeof payload !== "object" || Array.isArray(payload)) + return null; + const target = (payload as Record).target; + if (!target || typeof target !== "object" || Array.isArray(target)) + return null; + const record = target as Record; + if (record.type !== "issue_document") return null; + const revisionId = readStringFromRecord(record, "revisionId"); + const key = readStringFromRecord(record, "key"); + const issueId = readStringFromRecord(record, "issueId") ?? fallbackIssueId; + if (!revisionId || !key || !issueId) return null; + return { revisionId, key, issueId }; +} + +async function resolveAcceptedPlanClaimOwner(input: { + dbOrTx: Pick; + claim: Pick< + typeof issuePlanDecompositions.$inferSelect, + "ownerAgentId" | "ownerUserId" | "ownerRunId" + >; + actorAgentId?: string | null; + actorUserId?: string | null; + actorRunId?: string | null; +}) { + const nextOwner = { + ownerAgentId: input.actorAgentId ?? null, + ownerUserId: input.actorUserId ?? null, + ownerRunId: input.actorRunId ?? null, + }; + if ( + input.claim.ownerAgentId === nextOwner.ownerAgentId && + input.claim.ownerUserId === nextOwner.ownerUserId && + input.claim.ownerRunId === nextOwner.ownerRunId + ) { + return nextOwner; + } + + if (!input.claim.ownerRunId) { + return nextOwner; + } + + const existingOwnerRun = await input.dbOrTx + .select({ status: heartbeatRuns.status }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, input.claim.ownerRunId)) + .then((rows) => rows[0] ?? null); + if ( + existingOwnerRun && + !TERMINAL_HEARTBEAT_RUN_STATUSES.has(existingOwnerRun.status) + ) { + return { + ownerAgentId: input.claim.ownerAgentId, + ownerUserId: input.claim.ownerUserId, + ownerRunId: input.claim.ownerRunId, + }; + } + + return nextOwner; +} + +async function findAcceptedPlanDocumentInteraction( + dbOrTx: Pick, + input: { + companyId: string; + sourceIssueId: string; + acceptedPlanRevisionId: string; + }, +): Promise { + const rows = await dbOrTx + .select({ + id: issueThreadInteractions.id, + payload: issueThreadInteractions.payload, + }) + .from(issueThreadInteractions) + .where( + and( + eq(issueThreadInteractions.companyId, input.companyId), + eq(issueThreadInteractions.issueId, input.sourceIssueId), + eq(issueThreadInteractions.kind, "request_confirmation"), + eq(issueThreadInteractions.status, "accepted"), + ), + ) + .orderBy( + desc(issueThreadInteractions.resolvedAt), + desc(issueThreadInteractions.createdAt), + ); + + for (const row of rows) { + const target = readAcceptedPlanConfirmationTarget( + row.payload, + input.sourceIssueId, + ); + if ( + target?.issueId === input.sourceIssueId && + target.key === "plan" && + target.revisionId === input.acceptedPlanRevisionId + ) { + return { id: row.id }; + } + } + return null; +} + +function createIssueDependencyReadiness( + issueId: string, +): IssueDependencyReadiness { + return { + issueId, + blockerIssueIds: [], + unresolvedBlockerIssueIds: [], + unresolvedBlockerCount: 0, + pendingFinalizeBlockerIssueIds: [], + allBlockersDone: true, + isDependencyReady: true, + }; +} + +/** + * Returns the set of execution-workspace ids whose most recent workspace operation + * is NOT a successful `workspace_finalize`. These workspaces have either an in-flight + * run, a failed finalize, or never reached the finalize barrier — dependents that + * read this workspace must wait until finalize succeeds. + * + * Workspaces with no recorded operations are considered finalized (nothing has + * touched them since they were realized). + */ +export async function listUnfinalizedExecutionWorkspaceIds( + dbOrTx: Pick, + companyId: string, + executionWorkspaceIds: string[], +): Promise> { + const unfinalized = new Set(); + if (executionWorkspaceIds.length === 0) return unfinalized; + + // Pull every workspace op for the candidate workspaces and pick the latest per + // workspace in memory. Per-workspace LATERAL queries would be tighter, but the + // candidate set is tiny in practice (one workspace per blocker per readiness call). + const rows = await dbOrTx + .select({ + executionWorkspaceId: workspaceOperations.executionWorkspaceId, + phase: workspaceOperations.phase, + status: workspaceOperations.status, + startedAt: workspaceOperations.startedAt, + }) + .from(workspaceOperations) + .where( + and( + eq(workspaceOperations.companyId, companyId), + inArray( + workspaceOperations.executionWorkspaceId, + executionWorkspaceIds, + ), + ), + ); + + const latestByWorkspace = new Map< + string, + { phase: string; status: string; startedAt: Date } + >(); + for (const row of rows) { + if (!row.executionWorkspaceId) continue; + const current = latestByWorkspace.get(row.executionWorkspaceId); + if (!current || row.startedAt > current.startedAt) { + latestByWorkspace.set(row.executionWorkspaceId, { + phase: row.phase, + status: row.status, + startedAt: row.startedAt, + }); + } + } + + for (const workspaceId of executionWorkspaceIds) { + const latest = latestByWorkspace.get(workspaceId); + if (!latest) continue; // no ops recorded → treat as finalized + if (latest.phase === "workspace_finalize" && latest.status === "succeeded") + continue; + unfinalized.add(workspaceId); + } + + return unfinalized; +} + +async function listPendingFinalizeBlockerIssueIds( + dbOrTx: Pick, + companyId: string, + blockerWorkspacePairs: Array<{ + blockerIssueId: string; + executionWorkspaceId: string; + }>, + lockSources = false, +): Promise> { + const pending = new Set(); + const blockerIssueIds = [ + ...new Set(blockerWorkspacePairs.map((pair) => pair.blockerIssueId)), + ]; + const executionWorkspaceIds = [ + ...new Set(blockerWorkspacePairs.map((pair) => pair.executionWorkspaceId)), + ]; + if (blockerIssueIds.length === 0 || executionWorkspaceIds.length === 0) + return pending; + const blockerWorkspaceKeys = new Set( + blockerWorkspacePairs.map( + (pair) => `${pair.blockerIssueId}:${pair.executionWorkspaceId}`, + ), + ); + + if (lockSources) { + + + await dbOrTx.select({ id: executionWorkspaces.id }).from(executionWorkspaces) + .where(and(eq(executionWorkspaces.companyId, companyId), inArray(executionWorkspaces.id, executionWorkspaceIds))) + .for("update", { noWait: true }); + } + const query = dbOrTx + .select({ + issueId: workspaceOperations.issueId, + executionWorkspaceId: workspaceOperations.executionWorkspaceId, + phase: workspaceOperations.phase, + status: workspaceOperations.status, + startedAt: workspaceOperations.startedAt, + }) + .from(workspaceOperations) + .where( + and( + eq(workspaceOperations.companyId, companyId), + inArray( + workspaceOperations.executionWorkspaceId, + executionWorkspaceIds, + ), + ), + ); + + const rows = await (lockSources ? query.for("share", { noWait: true }) : query); + + const latestAttributedByBlockerWorkspace = new Map< + string, + { phase: string; status: string; startedAt: Date } + >(); + const latestUnattributedByWorkspace = new Map< + string, + { phase: string; status: string; startedAt: Date } + >(); + const latestSuccessfulFinalizeByWorkspace = new Map(); + for (const row of rows) { + if (!row.executionWorkspaceId) continue; + if (row.phase === "workspace_finalize" && row.status === "succeeded") { + const current = latestSuccessfulFinalizeByWorkspace.get( + row.executionWorkspaceId, + ); + if (!current || row.startedAt > current) { + latestSuccessfulFinalizeByWorkspace.set( + row.executionWorkspaceId, + row.startedAt, + ); + } + } + if (row.issueId) { + const key = `${row.issueId}:${row.executionWorkspaceId}`; + if (!blockerWorkspaceKeys.has(key)) continue; + const current = latestAttributedByBlockerWorkspace.get(key); + if (!current || row.startedAt > current.startedAt) { + latestAttributedByBlockerWorkspace.set(key, { + phase: row.phase, + status: row.status, + startedAt: row.startedAt, + }); + } + continue; + } + + const current = latestUnattributedByWorkspace.get(row.executionWorkspaceId); + if (!current || row.startedAt > current.startedAt) { + latestUnattributedByWorkspace.set(row.executionWorkspaceId, { + phase: row.phase, + status: row.status, + startedAt: row.startedAt, + }); + } + } + + for (const pair of blockerWorkspacePairs) { + const latest = + latestAttributedByBlockerWorkspace.get( + `${pair.blockerIssueId}:${pair.executionWorkspaceId}`, + ) ?? latestUnattributedByWorkspace.get(pair.executionWorkspaceId); + if (!latest) continue; // no ops recorded -> nothing to finalize for this blocker + if (latest.phase === "workspace_finalize" && latest.status === "succeeded") + continue; + const laterSuccessfulFinalize = latestSuccessfulFinalizeByWorkspace.get( + pair.executionWorkspaceId, + ); + if (laterSuccessfulFinalize && laterSuccessfulFinalize > latest.startedAt) + continue; + pending.add(pair.blockerIssueId); + } + + return pending; +} + +/** + * Whether a heartbeat run has reached a terminal state or no longer exists. + * A terminal/missing run can make no further progress on its execution + * workspace, so callers must not wait on it to advance an in-flight operation. + */ +export async function heartbeatRunIsTerminalOrMissing( + dbOrTx: Pick, + runId: string, +): Promise { + const run = await dbOrTx + .select({ status: heartbeatRuns.status }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)) + .then((rows: Array<{ status: string }>) => rows[0] ?? null); + if (!run) return true; + return TERMINAL_HEARTBEAT_RUN_STATUSES.has(run.status); +} + +/** + * Returns whether a specific run's sync-back on a specific execution workspace + * has settled — i.e. the accept/review gates that guard against a still-in-flight + * worktree sync no longer need to block on this run. + * + * Semantics: + * - No operations recorded → settled. The run never touched the workspace state + * the gates protect. + * - Earlier phases recorded but no `workspace_finalize` yet → NOT settled. The + * sync-back hasn't been attempted; the gate should wait for it. + * - Latest `workspace_finalize` reached a terminal status (`succeeded`, `failed`, + * or `skipped`) → settled. A finalize that ran and finished is done even if it + * failed: it will not retry within this run, so continuing to block would wedge + * the gate forever — a failed sync-back must not permanently block a + * confirmation accept behind a misleading "still syncing" error. + * - Latest `workspace_finalize` is still `running` → in flight, so NOT settled — + * unless the owning run has itself ended, in which case the `running` record is + * stale (the process died mid-finalize) and we treat it as settled rather than + * wait on a run that can never make progress. + */ +export async function runWorkspaceIsFinalized( + dbOrTx: Pick, + companyId: string, + executionWorkspaceId: string, + runId: string, +): Promise { + const rows = await dbOrTx + .select({ + phase: workspaceOperations.phase, + status: workspaceOperations.status, + startedAt: workspaceOperations.startedAt, + }) + .from(workspaceOperations) + .where( + and( + eq(workspaceOperations.companyId, companyId), + eq(workspaceOperations.executionWorkspaceId, executionWorkspaceId), + eq(workspaceOperations.heartbeatRunId, runId), + ), + ); + + if (rows.length === 0) return true; + + let latestFinalize: { status: string; startedAt: Date } | null = null; + for (const row of rows) { + if (row.phase !== "workspace_finalize") continue; + if (!latestFinalize || row.startedAt > latestFinalize.startedAt) + latestFinalize = row; + } + + // The run touched the workspace but hasn't reached the sync-back phase yet. + if (!latestFinalize) return false; + + // A finalize that reached any terminal status is settled — including `failed` + // and `skipped`. It will not retry within this run, so gates must stop waiting. + if (latestFinalize.status !== "running") return true; + + // Finalize is still marked `running`. It is only genuinely in flight while the + // owning run is alive; a `running` record left behind by an ended run is stale. + return heartbeatRunIsTerminalOrMissing(dbOrTx, runId); +} + +async function listIssueDependencyReadinessMap( + dbOrTx: Pick, + companyId: string, + issueIds: string[], + lockSources = false, +) { + const uniqueIssueIds = [...new Set(issueIds.filter(Boolean))]; + const readinessMap = new Map(); + for (const issueId of uniqueIssueIds) { + readinessMap.set(issueId, createIssueDependencyReadiness(issueId)); + } + if (uniqueIssueIds.length === 0) return readinessMap; + + const blockerQuery = dbOrTx + .select({ + issueId: issueRelations.relatedIssueId, + blockerIssueId: issueRelations.issueId, + blockerStatus: issues.status, + blockerExecutionWorkspaceId: issues.executionWorkspaceId, + }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.type, "blocks"), + inArray(issueRelations.relatedIssueId, uniqueIssueIds), + ), + ); + + + + const blockerRows = await (lockSources ? blockerQuery.for("update", { noWait: true }) : blockerQuery); + + // Collect issue/workspace pairs of "done" blockers — these are the only ones + // subject to the workspace-finalize barrier. Blockers that aren't done already + // mark the dependent as not-ready and don't need a finalize check. + const doneBlockerWorkspacePairs: Array<{ + blockerIssueId: string; + executionWorkspaceId: string; + }> = []; + for (const row of blockerRows) { + if (row.blockerStatus === "done" && row.blockerExecutionWorkspaceId) { + doneBlockerWorkspacePairs.push({ + blockerIssueId: row.blockerIssueId, + executionWorkspaceId: row.blockerExecutionWorkspaceId, + }); + } + } + const pendingFinalizeBlockerIssueIds = + await listPendingFinalizeBlockerIssueIds( + dbOrTx, + companyId, + doneBlockerWorkspacePairs, + lockSources, + ); + + for (const row of blockerRows) { + const current = + readinessMap.get(row.issueId) ?? + createIssueDependencyReadiness(row.issueId); + current.blockerIssueIds.push(row.blockerIssueId); + // Only done blockers resolve dependents; cancelled blockers stay unresolved + // until an operator removes or replaces the blocker relationship explicitly. + if (row.blockerStatus !== "done") { + current.unresolvedBlockerIssueIds.push(row.blockerIssueId); + current.unresolvedBlockerCount += 1; + current.allBlockersDone = false; + current.isDependencyReady = false; + } else if ( + row.blockerExecutionWorkspaceId && + pendingFinalizeBlockerIssueIds.has(row.blockerIssueId) + ) { + // Workspace-finalize barrier: the blocker's most recent run on its + // execution workspace hasn't recorded a successful workspace_finalize. + // Treat the dependent as not-ready until sync-back lands (or the run + // finalizes); a subsequent finalize wake will re-evaluate readiness. + // `allBlockersDone` is cleared too so that callers using it as a + // proxy for "this dependent can proceed" still see the gate. + current.unresolvedBlockerIssueIds.push(row.blockerIssueId); + current.unresolvedBlockerCount += 1; + current.pendingFinalizeBlockerIssueIds.push(row.blockerIssueId); + current.allBlockersDone = false; + current.isDependencyReady = false; + } + readinessMap.set(row.issueId, current); + } + + return readinessMap; +} + +async function listUnresolvedBlockerDetails( + dbOrTx: Pick, + companyId: string, + unresolvedBlockerIssueIds: string[], + pendingFinalizeBlockerIssueIds: string[] = [], +) { + if (unresolvedBlockerIssueIds.length === 0) return []; + const pendingFinalizeIds = new Set(pendingFinalizeBlockerIssueIds); + const rows = await dbOrTx + .select({ + issueId: issues.id, + identifier: issues.identifier, + title: issues.title, + }) + .from(issues) + .where( + and( + eq(issues.companyId, companyId), + inArray(issues.id, unresolvedBlockerIssueIds), + ), + ); + const rowsById = new Map(rows.map((row) => [row.issueId, row])); + return unresolvedBlockerIssueIds.map((issueId) => { + const row = rowsById.get(issueId); + return { + issueId, + identifier: row?.identifier ?? null, + title: row?.title ?? null, + reason: pendingFinalizeIds.has(issueId) + ? ("pending_finalize" as const) + : ("not_done" as const), + }; + }); +} + +async function listUnresolvedBlockerIssueIds( + dbOrTx: Pick, + companyId: string, + blockerIssueIds: string[], +) { + const uniqueBlockerIssueIds = [...new Set(blockerIssueIds.filter(Boolean))]; + if (uniqueBlockerIssueIds.length === 0) return []; + return dbOrTx + .select({ id: issues.id }) + .from(issues) + .where( + and( + eq(issues.companyId, companyId), + inArray(issues.id, uniqueBlockerIssueIds), + // Cancelled blockers intentionally remain unresolved until the relation changes. + ne(issues.status, "done"), + ), + ) + .then((rows) => rows.map((row) => row.id)); +} +async function getProjectDefaultGoalId( + db: ProjectGoalReader, + companyId: string, + projectId: string | null | undefined, +) { + if (!projectId) return null; + const row = await db + .select({ goalId: projects.goalId }) + .from(projects) + .where(and(eq(projects.id, projectId), eq(projects.companyId, companyId))) + .then((rows) => rows[0] ?? null); + return row?.goalId ?? null; +} + +async function getWorkspaceInheritanceIssue( + db: DbReader, + companyId: string, + issueId: string, +) { + const issue = await db + .select({ + id: issues.id, + projectId: issues.projectId, + projectWorkspaceId: issues.projectWorkspaceId, + executionWorkspaceId: issues.executionWorkspaceId, + executionWorkspaceSettings: issues.executionWorkspaceSettings, + }) + .from(issues) + .where(and(eq(issues.id, issueId), eq(issues.companyId, companyId))) + .then((rows) => rows[0] ?? null); + if (!issue) { + throw notFound("Workspace inheritance issue not found"); + } + return issue; +} + +function participatedByAgentCondition(companyId: string, agentId: string) { + return sql` + ( + ${issues.createdByAgentId} = ${agentId} + OR ${issues.assigneeAgentId} = ${agentId} + OR EXISTS ( + SELECT 1 + FROM ${issueComments} + WHERE ${issueComments.issueId} = ${issues.id} + AND ${issueComments.companyId} = ${companyId} + AND ${issueComments.authorAgentId} = ${agentId} + ) + OR EXISTS ( + SELECT 1 + FROM ${activityLog} + WHERE ${activityLog.companyId} = ${companyId} + AND ${activityLog.entityType} = 'issue' + AND ${activityLog.entityId} = ${issues.id}::text + AND ${activityLog.agentId} = ${agentId} + ) + ) + `; +} + +function myLastCommentAtExpr(companyId: string, userId: string) { + return sql` + ( + SELECT MAX(${issueComments.createdAt}) + FROM ${issueComments} + WHERE ${issueComments.issueId} = ${issues.id} + AND ${issueComments.companyId} = ${companyId} + AND ${issueComments.authorUserId} = ${userId} + AND ${directHumanCommentCondition()} + ) + `; +} + +function myLastReadAtExpr(companyId: string, userId: string) { + return sql` + ( + SELECT MAX(${issueReadStates.lastReadAt}) + FROM ${issueReadStates} + WHERE ${issueReadStates.issueId} = ${issues.id} + AND ${issueReadStates.companyId} = ${companyId} + AND ${issueReadStates.userId} = ${userId} + ) + `; +} + +function myLastTouchAtExpr(companyId: string, userId: string) { + const myLastCommentAt = myLastCommentAtExpr(companyId, userId); + const myLastReadAt = myLastReadAtExpr(companyId, userId); + return sql` + GREATEST( + COALESCE(${myLastCommentAt}, to_timestamp(0)), + COALESCE(${myLastReadAt}, to_timestamp(0)), + COALESCE(CASE WHEN ${issues.createdByUserId} = ${userId} AND ${humanCreatedIssueCondition()} THEN ${issues.createdAt} ELSE NULL END, to_timestamp(0)), + COALESCE(CASE WHEN ${issues.assigneeUserId} = ${userId} THEN ${issues.updatedAt} ELSE NULL END, to_timestamp(0)) + ) + `; +} + +const ISSUE_LOCAL_INBOX_ACTIVITY_ACTIONS = [ + "issue.read_marked", + "issue.read_unmarked", + "issue.inbox_archived", + "issue.inbox_unarchived", +] as const; + +function issueLatestCommentAtExpr(companyId: string) { + return sql` + ( + SELECT MAX(${issueComments.createdAt}) + FROM ${issueComments} + WHERE ${issueComments.issueId} = ${issues.id} + AND ${issueComments.companyId} = ${companyId} + ) + `; +} + +function issueLatestLogAtExpr(companyId: string) { + return sql` + ( + SELECT MAX(${activityLog.createdAt}) + FROM ${activityLog} + WHERE ${activityLog.companyId} = ${companyId} + AND ${activityLog.entityType} = 'issue' + AND ${activityLog.entityId} = ${issues.id}::text + AND NOT ${automaticReadRecoveryCleanupCondition()} + AND ${activityLog.action} NOT IN (${sql.join( + ISSUE_LOCAL_INBOX_ACTIVITY_ACTIONS.map((action) => sql`${action}`), + sql`, `, + )}) + ) + `; +} + +function issueCanonicalLastActivityAtExpr(companyId: string) { + const latestCommentAt = issueLatestCommentAtExpr(companyId); + const latestLogAt = issueLatestLogAtExpr(companyId); + return sql` + GREATEST( + ${issues.updatedAt}, + COALESCE(${latestCommentAt}, to_timestamp(0)), + COALESCE(${latestLogAt}, to_timestamp(0)) + ) + `; +} + +function unreadForUserCondition(companyId: string, userId: string) { + const touchedCondition = touchedByUserCondition(companyId, userId); + const myLastTouchAt = myLastTouchAtExpr(companyId, userId); + const myLastRequestReadAt = sql`GREATEST( + COALESCE(${myLastCommentAtExpr(companyId, userId)}, to_timestamp(0)), + COALESCE(${myLastReadAtExpr(companyId, userId)}, to_timestamp(0)) + )`; + return sql` + ( + ${touchedCondition} + AND (EXISTS ( + SELECT 1 + FROM ${issueComments} + WHERE ${issueComments.issueId} = ${issues.id} + AND ${issueComments.companyId} = ${companyId} + AND ( + ${issueComments.authorUserId} IS NULL + OR ${issueComments.authorUserId} <> ${userId} + ) + AND ${issueComments.createdAt} > ${myLastTouchAt} + ) OR ${pendingUserRequestCondition(companyId, userId, myLastRequestReadAt)}) + ) + `; +} + +function inboxVisibleForUserCondition(companyId: string, userId: string) { + return sql` + NOT EXISTS ( + SELECT 1 + FROM ${issueInboxArchives} + WHERE ${issueInboxArchives.issueId} = ${issues.id} + AND ${issueInboxArchives.companyId} = ${companyId} + AND ${issueInboxArchives.userId} = ${userId} + AND NOT ( + EXISTS ( + SELECT 1 + FROM ${issueThreadInteractions} + WHERE ${issueThreadInteractions.issueId} = ${issues.id} + AND ${issueThreadInteractions.companyId} = ${companyId} + AND ${issueThreadInteractions.kind} IN ( + 'suggest_tasks', + 'ask_user_questions', + 'request_confirmation' + ) + AND ${issueThreadInteractions.createdAt} > ${issueInboxArchives.archivedAt} + ) + OR ${pendingUserRequestCondition(companyId, userId, sql`${issueInboxArchives.archivedAt}`)} + OR EXISTS ( + SELECT 1 + FROM ${activityLog} + WHERE ${activityLog.companyId} = ${companyId} + AND ${activityLog.entityType} = 'issue' + AND ${activityLog.entityId} = ${issues.id}::text + AND ${activityLog.action} = 'issue.updated' + AND ${activityLog.createdAt} > ${issueInboxArchives.archivedAt} + AND ${activityLog.details}->>'status' IN ('in_review', 'blocked', 'done') + AND ${activityLog.details}->'_previous'->>'status' + IS DISTINCT FROM ${activityLog.details}->>'status' + AND NOT ( + ${activityLog.details}->>'status' = 'done' + AND ${issues.completedAt} IS NOT NULL + AND ${issueInboxArchives.archivedAt} >= ${issues.completedAt} + ) + ) + OR EXISTS ( + SELECT 1 + FROM ${issueComments} + WHERE ${issueComments.issueId} = ${issues.id} + AND ${issueComments.companyId} = ${companyId} + AND ${issueComments.createdAt} > ${issueInboxArchives.archivedAt} + AND ${issueComments.deletedAt} IS NULL + AND ( + ( + ${issueComments.authorUserId} IS NOT NULL + AND ${issueComments.authorUserId} <> ${userId} + AND ${issueComments.authorAgentId} IS NULL + AND ${issueComments.derivedAuthorAgentId} IS NULL + ) + OR POSITION(${`](user://${userId})`} IN ${issueComments.body}) > 0 + ) + ) + ) + ) + `; +} + +const LEGACY_PLUGIN_OPERATION_ORIGIN_KINDS = [ + "plugin:paperclipai.content-machine:case", + "plugin:paperclipai.content-machine:evaluation", + "plugin:paperclipai.content-machine:source-sync", +] as const; + +function nonPluginOperationIssueCondition() { + return sql`NOT ( + ${issues.originKind} LIKE 'plugin:%:operation' + OR ${issues.originKind} LIKE 'plugin:%:operation:%' + OR ${inArray(issues.originKind, LEGACY_PLUGIN_OPERATION_ORIGIN_KINDS)} + )`; +} + +function shouldIncludePluginOperationIssues(filters: IssueFilters | undefined) { + return Boolean( + filters?.includePluginOperations || + filters?.originKind || + filters?.originKindPrefix || + filters?.originId || + filters?.projectId, + ); +} + +export function deriveIssueUserContext( + issue: IssueUserContextInput, + userId: string, + stats: + | { + isUnreadForMe?: boolean; + myLastCommentAt: Date | string | null; + myLastReadAt: Date | string | null; + lastExternalCommentAt: Date | string | null; + } + | null + | undefined, +) { + const normalizeDate = (value: Date | string | null | undefined) => { + if (!value) return null; + if (value instanceof Date) + return Number.isNaN(value.getTime()) ? null : value; + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? null : parsed; + }; + + const myLastCommentAt = normalizeDate(stats?.myLastCommentAt); + const myLastReadAt = normalizeDate(stats?.myLastReadAt); + const createdTouchAt = + issue.createdByUserId === userId && !isAgentCreatedIssue(issue) + ? normalizeDate(issue.createdAt) + : null; + const assignedTouchAt = + issue.assigneeUserId === userId ? normalizeDate(issue.updatedAt) : null; + const myLastTouchAt = + [myLastCommentAt, myLastReadAt, createdTouchAt, assignedTouchAt] + .filter((value): value is Date => value instanceof Date) + .sort((a, b) => b.getTime() - a.getTime())[0] ?? null; + const lastExternalCommentAt = normalizeDate(stats?.lastExternalCommentAt); + const isUnreadForMe = stats?.isUnreadForMe ?? Boolean( + myLastTouchAt && + lastExternalCommentAt && + lastExternalCommentAt.getTime() > myLastTouchAt.getTime(), + ); + + return { + myLastTouchAt, + lastExternalCommentAt, + isUnreadForMe, + }; +} + +function latestIssueActivityAt( + ...values: Array +): Date | null { + const normalized = values + .map((value) => { + if (!value) return null; + if (value instanceof Date) + return Number.isNaN(value.getTime()) ? null : value; + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? null : parsed; + }) + .filter((value): value is Date => value instanceof Date) + .sort((a, b) => b.getTime() - a.getTime()); + return normalized[0] ?? null; +} + +type InboxArchiveAttributionRow = { + issueId: string; + archivedAt: Date; + archivedByActorType: "user" | "agent"; + archivedByAgentId: string | null; + archivedByRunId: string | null; +}; + +async function inboxArchiveRowsForIssues( + dbOrTx: Db, + companyId: string, + userId: string, + issueIds: string[], +): Promise { + if (issueIds.length === 0) return []; + return dbOrTx + .select({ + issueId: issueInboxArchives.issueId, + archivedAt: issueInboxArchives.archivedAt, + archivedByActorType: issueInboxArchives.archivedByActorType, + archivedByAgentId: issueInboxArchives.archivedByAgentId, + archivedByRunId: issueInboxArchives.archivedByRunId, + }) + .from(issueInboxArchives) + .where( + and( + eq(issueInboxArchives.companyId, companyId), + eq(issueInboxArchives.userId, userId), + inArray(issueInboxArchives.issueId, issueIds), + ), + ); +} + +function activeInboxArchiveFields( + archive: InboxArchiveAttributionRow | undefined, + lastActivityAt: Date, +) { + if (!archive || archive.archivedAt.getTime() < lastActivityAt.getTime()) + return {}; + return { + archivedAt: archive.archivedAt, + archivedByActorType: archive.archivedByActorType, + archivedByAgentId: archive.archivedByAgentId, + archivedByRunId: archive.archivedByRunId, + }; +} + +function issueListOrderBy( + companyId: string, + { + hasSearch, + priorityOrder, + searchOrder, + sortField, + sortDir, + }: { + hasSearch: boolean; + priorityOrder: SQL; + searchOrder: SQL; + sortField?: IssueFilters["sortField"]; + sortDir?: IssueFilters["sortDir"]; + }, +) { + if (sortField === "id") return [sortDir === "desc" ? desc(issues.id) : asc(issues.id)]; + const canonicalLastActivityAt = issueCanonicalLastActivityAtExpr(companyId); + if (sortField === "updated") { + const activityOrder = + sortDir === "asc" + ? asc(canonicalLastActivityAt) + : desc(canonicalLastActivityAt); + const updatedOrder = + sortDir === "asc" ? asc(issues.updatedAt) : desc(issues.updatedAt); + const idOrder = sortDir === "asc" ? asc(issues.id) : desc(issues.id); + return hasSearch + ? [asc(searchOrder), activityOrder, updatedOrder, idOrder] + : [activityOrder, updatedOrder, idOrder]; + } + + return [ + hasSearch ? asc(searchOrder) : asc(priorityOrder), + asc(priorityOrder), + desc(canonicalLastActivityAt), + desc(issues.updatedAt), + desc(issues.id), + ]; +} + +async function labelMapForIssues( + dbOrTx: any, + issueIds: string[], +): Promise> { + const map = new Map(); + if (issueIds.length === 0) return map; + for (const issueIdChunk of chunkList( + issueIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const rows = await dbOrTx + .select({ + issueId: issueLabels.issueId, + label: labels, + }) + .from(issueLabels) + .innerJoin(labels, eq(issueLabels.labelId, labels.id)) + .where(inArray(issueLabels.issueId, issueIdChunk)) + .orderBy(asc(labels.name), asc(labels.id)); + + for (const row of rows) { + const existing = map.get(row.issueId); + if (existing) existing.push(row.label); + else map.set(row.issueId, [row.label]); + } + } + return map; +} + +async function withIssueLabels( + dbOrTx: any, + rows: IssueRow[], +): Promise { + if (rows.length === 0) return []; + const issueIds = rows.map((row) => row.id); + const [labelsByIssueId, watchdogByIssueId] = await Promise.all([ + labelMapForIssues(dbOrTx, issueIds), + watchdogMapForIssues(dbOrTx, rows), + ]); + return rows.map((row) => { + const issueLabels = labelsByIssueId.get(row.id) ?? []; + return { + ...row, + labels: issueLabels, + labelIds: issueLabels.map((label) => label.id), + watchdog: watchdogByIssueId.get(row.id) ?? null, + }; + }); +} + +async function watchdogMapForIssues( + dbOrTx: any, + rows: IssueRow[], +): Promise> { + const map = new Map(); + if (rows.length === 0) return map; + const byCompany = new Map(); + for (const row of rows) { + const ids = byCompany.get(row.companyId) ?? []; + ids.push(row.id); + byCompany.set(row.companyId, ids); + } + for (const [companyId, issueIds] of byCompany.entries()) { + for (const issueIdChunk of chunkList( + [...new Set(issueIds)], + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const watchdogRows = await dbOrTx + .select() + .from(issueWatchdogs) + .where( + and( + eq(issueWatchdogs.companyId, companyId), + inArray(issueWatchdogs.issueId, issueIdChunk), + eq(issueWatchdogs.status, "active"), + ), + ); + for (const row of watchdogRows) { + map.set(row.issueId, summarizeIssueWatchdog(row)); + } + } + } + return map; +} + +const ACTIVE_RUN_STATUSES = ["queued", "running"]; +const BLOCKER_ATTENTION_ACTIVE_RUN_STATUSES = ["queued", "running"]; +const BLOCKER_ATTENTION_ACTIVE_WAKE_STATUSES = [ + "queued", + "deferred_issue_execution", +]; +const BLOCKER_ATTENTION_PENDING_INTERACTION_STATUSES = ["pending"]; +const BLOCKER_ATTENTION_PENDING_APPROVAL_STATUSES = [ + "pending", + "revision_requested", +]; +const BLOCKER_ATTENTION_OPEN_RECOVERY_ORIGIN_KIND = + "harness_liveness_escalation"; +const BLOCKER_ATTENTION_CHILD_TERMINAL_STATUSES = ["done", "cancelled"]; +function lowTrustBoundaryIssueCondition( + companyId: string, + boundary: (LowTrustBoundary & { companyId: string }) | null | undefined, +) { + if (!boundary || boundary.companyId !== companyId) return null; + const clauses: SQL[] = []; + const issueIds = [...new Set(boundary.issueIds ?? [])]; + const projectIds = [...new Set(boundary.projectIds ?? [])]; + if (issueIds.length > 0) clauses.push(inArray(issues.id, issueIds)); + if (projectIds.length > 0) + clauses.push(inArray(issues.projectId, projectIds)); + if (boundary.rootIssueId) { + clauses.push(sql` + ${issues.id} IN ( + WITH RECURSIVE descendants(id) AS ( + SELECT ${issues.id} + FROM ${issues} + WHERE ${issues.companyId} = ${companyId} + AND ${issues.id} = ${boundary.rootIssueId} + UNION + SELECT ${issues.id} + FROM ${issues} + JOIN descendants ON ${issues.parentId} = descendants.id + WHERE ${issues.companyId} = ${companyId} + ) + SELECT id FROM descendants + ) + `); + } + if (clauses.length === 0) return sql`false`; + return or(...clauses); +} + +const BLOCKER_ATTENTION_OPEN_RECOVERY_TERMINAL_STATUSES = ["done", "cancelled"]; +export const BLOCKER_ATTENTION_MAX_DEPTH = 8; +export const BLOCKER_ATTENTION_MAX_NODES = 2000; +const BLOCKER_ATTENTION_INVOKABLE_AGENT_STATUSES = new Set([ + "active", + "idle", + "running", + "error", +]); + +type IssueBlockerAttentionNode = { + id: string; + companyId: string; + parentId: string | null; + identifier: string | null; + title: string; + status: string; + executionRunId?: string | null; + assigneeAgentId: string | null; + assigneeUserId: string | null; +}; +type IssueBlockerAttentionInputNode = Pick< + IssueBlockerAttentionNode, + | "id" + | "companyId" + | "parentId" + | "identifier" + | "title" + | "status" + | "assigneeAgentId" + | "assigneeUserId" +> & { executionRunId?: string | null }; + +type IssueBlockerAttentionEdge = { + issueId: string; + blockerIssueId: string; +}; +type IssueBlockerAttentionQueryRow = IssueBlockerAttentionNode & { + issueId: string | null; + blockerIssueId: string; +}; +type IssueBlockerAttentionActivePathRow = { + issueId: string | null; +}; +type IssueBlockerAttentionAgentRow = { + id: string; + companyId: string; + status: string; +}; + +async function activeRunMapForIssues( + dbOrTx: any, + issueRows: IssueWithLabels[], +): Promise> { + const map = new Map(); + const runIds = issueRows + .map((row) => row.executionRunId) + .filter((id): id is string => id != null); + if (runIds.length === 0) return map; + + for (const runIdChunk of chunkList( + [...new Set(runIds)], + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const rows = await dbOrTx + .select({ + id: heartbeatRuns.id, + status: heartbeatRuns.status, + agentId: heartbeatRuns.agentId, + invocationSource: heartbeatRuns.invocationSource, + triggerDetail: heartbeatRuns.triggerDetail, + startedAt: heartbeatRuns.startedAt, + finishedAt: heartbeatRuns.finishedAt, + createdAt: heartbeatRuns.createdAt, + }) + .from(heartbeatRuns) + .where( + and( + inArray(heartbeatRuns.id, runIdChunk), + inArray(heartbeatRuns.status, ACTIVE_RUN_STATUSES), + ), + ); + + for (const row of rows) { + map.set(row.id, row); + } + } + for (const companyId of new Set(issueRows.map(row => row.companyId))) { + const scopedIds = issueRows.filter(row => row.companyId === companyId).flatMap(row => row.executionRunId && map.has(row.executionRunId) ? [row.executionRunId] : []); + const projections = await executionProjectionsForRuns(dbOrTx, companyId, scopedIds); + for (const [runId, execution] of projections) { + const row = map.get(runId); + if (row) row.execution = execution; + } + } + return map; +} + +async function liveDescendantCountMapForIssues( + dbOrTx: any, + companyId: string, + issueIds: string[], +): Promise> { + const uniqueIssueIds = [...new Set(issueIds)]; + const map = new Map(); + if (uniqueIssueIds.length === 0) return map; + + for (const issueIdChunk of chunkList( + uniqueIssueIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const targetRows = issueIdChunk.map((issueId) => sql`(${issueId}::uuid)`); + const rows = await dbOrTx.execute(sql<{ + issueId: string; + liveDescendantCount: number; + }>` + WITH RECURSIVE + target_issues(issue_id) AS ( + VALUES ${sql.join(targetRows, sql`, `)} + ), + live_issues(live_issue_id, parent_id) AS ( + SELECT DISTINCT live_issue.id, live_issue.parent_id + FROM issues live_issue + JOIN heartbeat_runs live_run ON live_run.id = live_issue.execution_run_id + WHERE live_issue.company_id = ${companyId} + AND live_issue.hidden_at IS NULL + AND live_issue.harness_kind IS NULL + AND live_run.company_id = ${companyId} + AND live_run.status IN ('queued', 'running') + UNION + SELECT DISTINCT live_issue.id, live_issue.parent_id + FROM heartbeat_runs live_run + JOIN issues live_issue ON live_issue.id::text = (live_run.context_snapshot ->> 'issueId') + WHERE live_issue.company_id = ${companyId} + AND live_issue.hidden_at IS NULL + AND live_issue.harness_kind IS NULL + AND live_run.company_id = ${companyId} + AND live_run.status IN ('queued', 'running') + ), + live_ancestors(live_issue_id, ancestor_id, next_parent_id, visited_issue_ids) AS ( + SELECT live_issues.live_issue_id, parent.id, parent.parent_id, ARRAY[live_issues.live_issue_id, parent.id] + FROM live_issues + JOIN issues parent ON parent.id = live_issues.parent_id + WHERE parent.company_id = ${companyId} + AND parent.hidden_at IS NULL + AND parent.harness_kind IS NULL + UNION ALL + SELECT + live_ancestors.live_issue_id, + parent.id, + parent.parent_id, + live_ancestors.visited_issue_ids || parent.id + FROM live_ancestors + JOIN issues parent ON parent.id = live_ancestors.next_parent_id + WHERE parent.company_id = ${companyId} + AND parent.hidden_at IS NULL + AND parent.harness_kind IS NULL + AND NOT parent.id = ANY(live_ancestors.visited_issue_ids) + ) + SELECT + live_ancestors.ancestor_id::text AS "issueId", + count(DISTINCT live_ancestors.live_issue_id)::int AS "liveDescendantCount" + FROM live_ancestors + JOIN target_issues ON target_issues.issue_id = live_ancestors.ancestor_id + WHERE live_ancestors.ancestor_id <> live_ancestors.live_issue_id + GROUP BY live_ancestors.ancestor_id + `); + + const resultRows = Array.isArray(rows) + ? rows + : Array.from(rows as Iterable); + for (const row of resultRows) { + if (typeof row !== "object" || row === null) continue; + const issueId = (row as { issueId?: unknown }).issueId; + const liveDescendantCount = (row as { liveDescendantCount?: unknown }) + .liveDescendantCount; + if (typeof issueId !== "string") continue; + const count = + typeof liveDescendantCount === "number" + ? liveDescendantCount + : Number(liveDescendantCount); + if (Number.isFinite(count)) map.set(issueId, count); + } + } + + return map; +} + +function createIssueBlockerAttention( + input: Partial = {}, +): IssueBlockerAttention { + return { + state: input.state ?? "none", + reason: input.reason ?? null, + unresolvedBlockerCount: input.unresolvedBlockerCount ?? 0, + coveredBlockerCount: input.coveredBlockerCount ?? 0, + stalledBlockerCount: input.stalledBlockerCount ?? 0, + attentionBlockerCount: input.attentionBlockerCount ?? 0, + pendingFinalizeBlockerIssueIds: input.pendingFinalizeBlockerIssueIds ?? [], + sampleBlockerIdentifier: input.sampleBlockerIdentifier ?? null, + sampleStalledBlockerIdentifier: + input.sampleStalledBlockerIdentifier ?? null, + blockingTreeLive: input.blockingTreeLive ?? false, + directBlockerIssueId: input.directBlockerIssueId ?? null, + terminalBlockerIssueId: input.terminalBlockerIssueId ?? null, + terminalBlocker: input.terminalBlocker ?? null, + }; +} + +function blockerSampleIdentifier( + node: IssueBlockerAttentionNode | null | undefined, +) { + return node?.identifier ?? node?.id ?? null; +} + +function appendBlockerAttentionEdges( + edgesByIssueId: Map, + rows: IssueBlockerAttentionEdge[], +) { + for (const row of rows) { + const existing = edgesByIssueId.get(row.issueId) ?? []; + if (!existing.some((edge) => edge.blockerIssueId === row.blockerIssueId)) { + existing.push(row); + edgesByIssueId.set(row.issueId, existing); + } + } +} + +type IssueRelationSummaryRow = { + relatedId: string; + identifier: string | null; + title: string; + status: string; + priority: string; + assigneeAgentId: string | null; + assigneeUserId: string | null; +}; + +function summarizeIssueRelationRow( + row: IssueRelationSummaryRow, +): IssueRelationIssueSummary { + return { + id: row.relatedId, + identifier: row.identifier, + title: row.title, + status: row.status as IssueRelationIssueSummary["status"], + priority: row.priority as IssueRelationIssueSummary["priority"], + assigneeAgentId: row.assigneeAgentId, + assigneeUserId: row.assigneeUserId, + }; +} + +/** Inline draft images become task attachments in the same transaction as publication. */ +async function attachOwnedDraftImages(tx: DbTransaction, issue: typeof issues.$inferSelect, body: string | null | undefined, + actor: { userId?: string | null; agentId?: string | null }) { + const ids = [...new Set(Array.from((body ?? "").matchAll(/\/api\/assets\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\/content/gi), match => match[1]!))]; + if (!ids.length || (!actor.userId && !actor.agentId)) return; + const owned = await tx.select().from(assets).where(and(eq(assets.companyId, issue.companyId), inArray(assets.id, ids), + actor.userId ? eq(assets.createdByUserId, actor.userId) : eq(assets.createdByAgentId, actor.agentId!))); + for (const asset of owned) { + if (!asset.objectKey.startsWith(`${issue.companyId}/assets/issues/drafts/`)) continue; + await tx.insert(issueAttachments).values({ companyId: issue.companyId, issueId: issue.id, assetId: asset.id }).onConflictDoNothing(); + } +} + +export async function ensureAssignmentIssueAccessGrant( + dbOrTx: any, + issue: typeof issues.$inferSelect, + previous: Pick | null, + actor: { agentId?: string | null; userId?: string | null }, +) { + if (issue.visibility !== "private") return null; + const subject = issue.assigneeAgentId + ? { subjectType: "agent" as const, subjectId: issue.assigneeAgentId } + : issue.assigneeUserId + ? { subjectType: "user" as const, subjectId: issue.assigneeUserId } + : null; + if (!subject) return null; + if ( + previous + && previous.visibility === "private" + && previous.assigneeAgentId === issue.assigneeAgentId + && previous.assigneeUserId === issue.assigneeUserId + ) return null; + + const grantIssueId = issue.id; + const lockKey = `issue-access-grant:${grantIssueId}:${subject.subjectType}:${subject.subjectId}`; + await dbOrTx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${lockKey}, 0))`); + const active = await dbOrTx + .select({ id: issueAccessGrants.id }) + .from(issueAccessGrants) + .where(and( + eq(issueAccessGrants.issueId, grantIssueId), + eq(issueAccessGrants.subjectType, subject.subjectType), + eq(issueAccessGrants.subjectId, subject.subjectId), + isNull(issueAccessGrants.revokedAt), + )) + .limit(1) + .then((rows: Array<{ id: string }>) => rows[0] ?? null); + if (active) return null; + + const grant = await dbOrTx + .insert(issueAccessGrants) + .values({ + issueId: grantIssueId, + ...subject, + source: "assignment", + grantedByUserId: actor.userId ?? null, + grantedByAgentId: actor.agentId ?? null, + }) + .returning() + .then((rows: Array) => rows[0]!); + await logActivity(dbOrTx as Db, { + companyId: issue.companyId, + actorType: actor.agentId ? "agent" : actor.userId ? "user" : "system", + actorId: actor.agentId ?? actor.userId ?? "issue_service", + agentId: actor.agentId ?? null, + action: "issue_access_grant.created", + entityType: "issue_access_grant", + entityId: grant.id, + details: { + issueId: grant.issueId, + subjectType: grant.subjectType, + subjectId: grant.subjectId, + source: grant.source, + }, + }); + return grant; +} + +async function terminalExplicitBlockersByRoot( + companyId: string, + roots: IssueRelationIssueSummary[], + dbOrTx: DbReader, +): Promise> { + const rootIds = [...new Set(roots.map((root) => root.id))]; + const terminalByRoot = new Map(); + if (rootIds.length === 0) return terminalByRoot; + + const nodesById = new Map(); + const edgesByIssueId = new Map(); + for (const root of roots) nodesById.set(root.id, root); + + let frontier = rootIds; + for ( + let depth = 0; + frontier.length > 0 && depth < BLOCKER_ATTENTION_MAX_DEPTH; + depth += 1 + ) { + const nextFrontier = new Set(); + for (const chunk of chunkList( + [...new Set(frontier)], + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const rows = await dbOrTx + .select({ + currentIssueId: issueRelations.relatedIssueId, + relatedId: issues.id, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + priority: issues.priority, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.type, "blocks"), + inArray(issueRelations.relatedIssueId, chunk), + eq(issues.companyId, companyId), + ne(issues.status, "done"), + ), + ); + + for (const row of rows) { + const existingEdges = edgesByIssueId.get(row.currentIssueId) ?? []; + if (!existingEdges.includes(row.relatedId)) { + existingEdges.push(row.relatedId); + edgesByIssueId.set(row.currentIssueId, existingEdges); + } + if (!nodesById.has(row.relatedId)) { + nodesById.set(row.relatedId, summarizeIssueRelationRow(row)); + nextFrontier.add(row.relatedId); + } + } + } + + if (nodesById.size > BLOCKER_ATTENTION_MAX_NODES) break; + frontier = [...nextFrontier]; + } + + const collectTerminal = ( + issueId: string, + seen: Set, + ): IssueRelationIssueSummary[] => { + if (seen.has(issueId)) return []; + const node = nodesById.get(issueId); + if (!node || node.status === "done") return []; + const nextSeen = new Set(seen); + nextSeen.add(issueId); + const downstreamIds = edgesByIssueId.get(issueId) ?? []; + if (downstreamIds.length === 0) return [node]; + return downstreamIds.flatMap((downstreamId) => + collectTerminal(downstreamId, nextSeen), + ); + }; + + for (const rootId of rootIds) { + const deduped = new Map(); + for (const blocker of collectTerminal(rootId, new Set())) { + if (blocker.id !== rootId) deduped.set(blocker.id, blocker); + } + if (deduped.size > 0) { + terminalByRoot.set( + rootId, + [...deduped.values()].sort((a, b) => a.title.localeCompare(b.title)), + ); + } + } + + return terminalByRoot; +} + +async function listIssueBlockerAttentionMap( + dbOrTx: any, + companyId: string, + issueRows: IssueBlockerAttentionInputNode[], +): Promise> { + const roots = issueRows.filter( + (row) => row.companyId === companyId && row.status === "blocked", + ); + const attentionMap = new Map(); + for (const row of issueRows) { + if (row.status !== "blocked") { + attentionMap.set(row.id, createIssueBlockerAttention()); + } + } + if (roots.length === 0) return attentionMap; + + const nodesById = new Map(); + const edgesByIssueId = new Map(); + for (const root of roots) nodesById.set(root.id, { ...root }); + + let frontier = roots.map((root) => root.id); + let truncated = false; + const pendingFinalizeBlockerIssueIds = new Set(); + for ( + let depth = 0; + frontier.length > 0 && depth < BLOCKER_ATTENTION_MAX_DEPTH; + depth += 1 + ) { + const nextFrontier = new Set(); + + for (const chunk of chunkList( + [...new Set(frontier)], + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const readinessByIssueId = await listIssueDependencyReadinessMap( + dbOrTx, + companyId, + chunk, + ); + for (const readiness of readinessByIssueId.values()) { + for (const blockerIssueId of readiness.pendingFinalizeBlockerIssueIds) { + pendingFinalizeBlockerIssueIds.add(blockerIssueId); + } + } + const explicitBlockerRowsPromise: Promise< + IssueBlockerAttentionQueryRow[] + > = dbOrTx + .select({ + issueId: issueRelations.relatedIssueId, + blockerIssueId: issues.id, + id: issues.id, + companyId: issues.companyId, + parentId: issues.parentId, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + executionRunId: issues.executionRunId, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.type, "blocks"), + inArray(issueRelations.relatedIssueId, chunk), + eq(issues.companyId, companyId), + ), + ); + const childRowsPromise: Promise = dbOrTx + .select({ + issueId: issues.parentId, + blockerIssueId: issues.id, + id: issues.id, + companyId: issues.companyId, + parentId: issues.parentId, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + executionRunId: issues.executionRunId, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + }) + .from(issues) + .where( + and( + eq(issues.companyId, companyId), + inArray(issues.parentId, chunk), + notInArray( + issues.status, + BLOCKER_ATTENTION_CHILD_TERMINAL_STATUSES, + ), + ), + ); + const [explicitBlockerRows, childRows] = await Promise.all([ + explicitBlockerRowsPromise, + childRowsPromise, + ]); + + const unresolvedExplicitBlockerRows = explicitBlockerRows.filter( + (row) => + row.status !== "done" || + pendingFinalizeBlockerIssueIds.has(row.blockerIssueId), + ); + appendBlockerAttentionEdges(edgesByIssueId, [ + ...unresolvedExplicitBlockerRows + .filter( + (row): row is IssueBlockerAttentionQueryRow & { issueId: string } => + row.issueId !== null, + ) + .map((row) => ({ + issueId: row.issueId, + blockerIssueId: row.blockerIssueId, + })), + ...childRows + .filter( + (row): row is IssueBlockerAttentionQueryRow & { issueId: string } => + row.issueId !== null, + ) + .map((row) => ({ + issueId: row.issueId, + blockerIssueId: row.blockerIssueId, + })), + ]); + + for (const row of [...unresolvedExplicitBlockerRows, ...childRows]) { + if (!row.issueId || nodesById.has(row.blockerIssueId)) continue; + nodesById.set(row.blockerIssueId, { + id: row.blockerIssueId, + companyId: row.companyId, + parentId: row.parentId, + identifier: row.identifier, + title: row.title, + status: row.status, + executionRunId: row.executionRunId, + assigneeAgentId: row.assigneeAgentId, + assigneeUserId: row.assigneeUserId, + }); + nextFrontier.add(row.blockerIssueId); + } + } + + if (nodesById.size > BLOCKER_ATTENTION_MAX_NODES) { + truncated = true; + break; + } + frontier = [...nextFrontier]; + } + if (frontier.length > 0) truncated = true; + + const nodeIds = [...nodesById.keys()]; + const activeIssueIds = new Set(); + const agentIds = new Set(); + const issueIdByExecutionRunId = new Map(); + for (const node of nodesById.values()) { + if (node.assigneeAgentId) agentIds.add(node.assigneeAgentId); + if (node.executionRunId) + issueIdByExecutionRunId.set(node.executionRunId, node.id); + } + + for (const chunk of chunkList( + [...issueIdByExecutionRunId.keys()], + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const runRows: Array<{ id: string }> = await dbOrTx + .select({ + id: heartbeatRuns.id, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.status, BLOCKER_ATTENTION_ACTIVE_RUN_STATUSES), + inArray(heartbeatRuns.id, chunk), + ), + ); + + for (const row of runRows) { + const issueId = issueIdByExecutionRunId.get(row.id); + if (issueId) activeIssueIds.add(issueId); + } + } + + for (const chunk of chunkList(nodeIds, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) { + const wakeRowsPromise: Promise = + dbOrTx + .select({ + issueId: sql< + string | null + >`${agentWakeupRequests.payload} ->> 'issueId'`, + }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + inArray( + agentWakeupRequests.status, + BLOCKER_ATTENTION_ACTIVE_WAKE_STATUSES, + ), + sql`${agentWakeupRequests.runId} is null`, + inArray( + sql`${agentWakeupRequests.payload} ->> 'issueId'`, + chunk, + ), + ), + ); + const wakeRows = await wakeRowsPromise; + for (const row of wakeRows) { + if (row.issueId) activeIssueIds.add(row.issueId); + } + } + + const explicitWaitCandidateIds = [...nodesById.values()] + .filter((node) => node.status !== "done") + .map((node) => node.id); + const explicitWaitingIssueIds = new Set(); + if (explicitWaitCandidateIds.length > 0) { + for (const chunk of chunkList( + explicitWaitCandidateIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const interactionRows: Array<{ issueId: string }> = await dbOrTx + .select({ issueId: issueThreadInteractions.issueId }) + .from(issueThreadInteractions) + .where( + and( + eq(issueThreadInteractions.companyId, companyId), + inArray( + issueThreadInteractions.status, + BLOCKER_ATTENTION_PENDING_INTERACTION_STATUSES, + ), + inArray(issueThreadInteractions.issueId, chunk), + ), + ); + for (const row of interactionRows) + explicitWaitingIssueIds.add(row.issueId); + + const approvalRows: Array<{ issueId: string }> = await dbOrTx + .select({ issueId: issueApprovals.issueId }) + .from(issueApprovals) + .innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id)) + .where( + and( + eq(issueApprovals.companyId, companyId), + inArray( + approvals.status, + BLOCKER_ATTENTION_PENDING_APPROVAL_STATUSES, + ), + inArray(issueApprovals.issueId, chunk), + ), + ); + for (const row of approvalRows) explicitWaitingIssueIds.add(row.issueId); + } + + // Recovery rows are intentionally company-wide: a liveness escalation for + // the same leaf blocker represents an active waiting path even when that + // blocker is reached through another blocked graph. + const recoveryRows: Array<{ id: string; originId: string | null }> = + await dbOrTx + .select({ id: issues.id, originId: issues.originId }) + .from(issues) + .where( + and( + eq(issues.companyId, companyId), + eq(issues.originKind, BLOCKER_ATTENTION_OPEN_RECOVERY_ORIGIN_KIND), + visibleIssueCondition(), + notInArray( + issues.status, + BLOCKER_ATTENTION_OPEN_RECOVERY_TERMINAL_STATUSES, + ), + ), + ); + for (const row of recoveryRows) { + const parsed = parseIssueGraphLivenessIncidentKey(row.originId); + if (!parsed || parsed.companyId !== companyId) continue; + explicitWaitingIssueIds.add(row.id); + explicitWaitingIssueIds.add(parsed.issueId); + explicitWaitingIssueIds.add(parsed.leafIssueId); + } + + const recoveryActionRows: Array<{ + id: string; + sourceIssueId: string; + status: string; + ownerType: string; + ownerAgentId: string | null; + ownerUserId: string | null; + }> = await dbOrTx + .select({ + id: issueRecoveryActions.id, + sourceIssueId: issueRecoveryActions.sourceIssueId, + status: issueRecoveryActions.status, + ownerType: issueRecoveryActions.ownerType, + ownerAgentId: issueRecoveryActions.ownerAgentId, + ownerUserId: issueRecoveryActions.ownerUserId, + }) + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, companyId), + inArray(issueRecoveryActions.status, ["active", "escalated"]), + inArray(issueRecoveryActions.sourceIssueId, explicitWaitCandidateIds), + ), + ); + const recoveryActionIds = recoveryActionRows.map((row) => row.id); + const liveRecoveryActionIds = new Set(); + for (const chunk of chunkList( + recoveryActionIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const [runRows, wakeRows] = await Promise.all([ + dbOrTx + .select({ + recoveryActionId: sql< + string | null + >`${heartbeatRuns.contextSnapshot} ->> 'recoveryActionId'`, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + inArray( + heartbeatRuns.status, + BLOCKER_ATTENTION_ACTIVE_RUN_STATUSES, + ), + inArray( + sql`${heartbeatRuns.contextSnapshot} ->> 'recoveryActionId'`, + chunk, + ), + ), + ), + dbOrTx + .select({ + recoveryActionId: sql< + string | null + >`${agentWakeupRequests.payload} ->> 'recoveryActionId'`, + }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + inArray( + agentWakeupRequests.status, + BLOCKER_ATTENTION_ACTIVE_WAKE_STATUSES, + ), + inArray( + sql`${agentWakeupRequests.payload} ->> 'recoveryActionId'`, + chunk, + ), + ), + ), + ]); + for (const row of [...runRows, ...wakeRows]) { + if (row.recoveryActionId) + liveRecoveryActionIds.add(row.recoveryActionId); + } + } + for (const row of recoveryActionRows) { + const healthy = + (row.status === "escalated" && row.ownerType === "board") || + Boolean(row.ownerUserId) || + (Boolean(row.ownerAgentId) && liveRecoveryActionIds.has(row.id)); + if (healthy) explicitWaitingIssueIds.add(row.sourceIssueId); + } + } + + const agentRows: IssueBlockerAttentionAgentRow[] = + agentIds.size > 0 + ? await dbOrTx + .select({ + id: agents.id, + companyId: agents.companyId, + status: agents.status, + }) + .from(agents) + .where( + and( + eq(agents.companyId, companyId), + inArray(agents.id, [...agentIds]), + ), + ) + : []; + const agentsById = new Map(agentRows.map((agent) => [agent.id, agent])); + + type PathClassification = { + covered: boolean; + stalled: boolean; + sampleBlockerIdentifier: string | null; + sampleStalledBlockerIdentifier: string | null; + terminalBlockerIssueId?: string | null; + }; + const classifyPath = ( + nodeId: string, + seen: Set, + ): PathClassification => { + const sample = blockerSampleIdentifier(nodesById.get(nodeId)); + if (truncated || seen.has(nodeId)) { + return { + covered: false, + stalled: false, + sampleBlockerIdentifier: sample, + sampleStalledBlockerIdentifier: null, + }; + } + const node = nodesById.get(nodeId); + if (!node || node.companyId !== companyId) { + return { + covered: false, + stalled: false, + sampleBlockerIdentifier: nodeId, + sampleStalledBlockerIdentifier: null, + }; + } + const nodeSample = blockerSampleIdentifier(node); + if ( + node.status === "done" && + !pendingFinalizeBlockerIssueIds.has(node.id) + ) { + return { + covered: true, + stalled: false, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: null, + }; + } + if (explicitWaitingIssueIds.has(node.id)) { + return { + covered: true, + stalled: false, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: null, + }; + } + if (node.assigneeUserId && node.status !== "cancelled") { + return { + covered: true, + stalled: false, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: null, + }; + } + if (node.status === "in_review") { + const hasWaitingPath = + activeIssueIds.has(node.id) || Boolean(node.assigneeUserId); + if (hasWaitingPath) { + return { + covered: true, + stalled: false, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: null, + }; + } + return { + covered: false, + stalled: true, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: nodeSample, + terminalBlockerIssueId: node.id, + }; + } + if (activeIssueIds.has(node.id)) { + return { + covered: true, + stalled: false, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: null, + }; + } + if (node.status === "cancelled") { + return { + covered: false, + stalled: false, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: null, + terminalBlockerIssueId: node.id, + }; + } + if (node.status === "backlog" && node.assigneeAgentId) { + return { + covered: false, + stalled: false, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: null, + terminalBlockerIssueId: node.id, + }; + } + + const downstream = (edgesByIssueId.get(node.id) ?? []).filter((edge) => { + const blocker = nodesById.get(edge.blockerIssueId); + return ( + blocker?.status !== "done" || + pendingFinalizeBlockerIssueIds.has(edge.blockerIssueId) + ); + }); + if (downstream.length > 0) { + const nextSeen = new Set(seen); + nextSeen.add(nodeId); + const classified = downstream.map((edge) => + classifyPath(edge.blockerIssueId, nextSeen), + ); + const stalledChild = classified.find( + (result) => result.stalled || result.sampleStalledBlockerIdentifier, + ); + const sampleStalled = + stalledChild?.sampleStalledBlockerIdentifier ?? null; + const hardAttention = + classified.find( + (result) => + !result.covered && !result.stalled && result.terminalBlockerIssueId, + ) ?? classified.find((result) => !result.covered && !result.stalled); + if (hardAttention) { + return { + covered: false, + stalled: false, + sampleBlockerIdentifier: hardAttention.sampleBlockerIdentifier, + sampleStalledBlockerIdentifier: sampleStalled, + terminalBlockerIssueId: hardAttention.terminalBlockerIssueId ?? null, + }; + } + const stalledEntry = classified.find((result) => result.stalled); + if (stalledEntry) { + return { + covered: false, + stalled: true, + sampleBlockerIdentifier: stalledEntry.sampleBlockerIdentifier, + sampleStalledBlockerIdentifier: sampleStalled, + terminalBlockerIssueId: stalledEntry.terminalBlockerIssueId ?? null, + }; + } + return { + covered: true, + stalled: false, + sampleBlockerIdentifier: + classified[0]?.sampleBlockerIdentifier ?? nodeSample, + sampleStalledBlockerIdentifier: null, + }; + } + + if (node.assigneeAgentId) { + const assignee = agentsById.get(node.assigneeAgentId); + if ( + !assignee || + assignee.companyId !== companyId || + !BLOCKER_ATTENTION_INVOKABLE_AGENT_STATUSES.has(assignee.status) + ) { + return { + covered: false, + stalled: false, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: null, + terminalBlockerIssueId: node.id, + }; + } + } + + return { + covered: false, + stalled: false, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: null, + terminalBlockerIssueId: node.id, + }; + }; + + const pathHasLiveWork = (nodeId: string, seen: Set): boolean => { + if (seen.has(nodeId)) return false; + const node = nodesById.get(nodeId); + if (!node || node.companyId !== companyId) return false; + if ( + node.status === "in_progress" || + activeIssueIds.has(node.id) || + explicitWaitingIssueIds.has(node.id) || + Boolean(node.assigneeUserId) + ) + return true; + + const nextSeen = new Set(seen); + nextSeen.add(nodeId); + return (edgesByIssueId.get(node.id) ?? []).some((edge) => { + const blocker = nodesById.get(edge.blockerIssueId); + if ( + blocker?.status === "done" && + !pendingFinalizeBlockerIssueIds.has(edge.blockerIssueId) + ) + return false; + return pathHasLiveWork(edge.blockerIssueId, nextSeen); + }); + }; + + const issueIdForSample = (sample: string | null | undefined) => { + if (!sample) return null; + for (const node of nodesById.values()) { + if (node.id === sample || node.identifier === sample) return node.id; + } + return null; + }; + + for (const root of roots) { + const topLevelEdges = (edgesByIssueId.get(root.id) ?? []).filter((edge) => { + const blocker = nodesById.get(edge.blockerIssueId); + return ( + blocker?.status !== "done" || + pendingFinalizeBlockerIssueIds.has(edge.blockerIssueId) + ); + }); + if (topLevelEdges.length === 0) { + attentionMap.set( + root.id, + createIssueBlockerAttention({ + state: "needs_attention", + reason: "attention_required", + terminalBlockerIssueId: root.id, + }), + ); + continue; + } + + const classified = topLevelEdges.map((edge) => ({ + edge, + result: classifyPath(edge.blockerIssueId, new Set([root.id])), + })); + const coveredBlockerCount = classified.filter( + (entry) => entry.result.covered, + ).length; + const stalledBlockerCount = classified.filter( + (entry) => entry.result.stalled, + ).length; + const attentionBlockerCount = + classified.length - coveredBlockerCount - stalledBlockerCount; + const hardAttentionEntry = + classified.find( + (entry) => + !entry.result.covered && + !entry.result.stalled && + entry.result.terminalBlockerIssueId, + ) ?? + classified.find( + (entry) => !entry.result.covered && !entry.result.stalled, + ); + const stalledEntry = + classified.find( + (entry) => entry.result.stalled && entry.result.terminalBlockerIssueId, + ) ?? classified.find((entry) => entry.result.stalled); + const sampleEntry = + hardAttentionEntry ?? stalledEntry ?? classified[0] ?? null; + const sampleNode = sampleEntry + ? nodesById.get(sampleEntry.edge.blockerIssueId) + : null; + const sampleStalledFromChain = classified + .map((entry) => entry.result.sampleStalledBlockerIdentifier) + .find((value) => value); + const sampledTerminalIdentifier = sampleEntry?.result.stalled + ? (sampleEntry.result.sampleStalledBlockerIdentifier ?? + sampleEntry.result.sampleBlockerIdentifier) + : (sampleEntry?.result.sampleBlockerIdentifier ?? + blockerSampleIdentifier(sampleNode)); + const terminalBlockerIssueId = + sampleEntry?.result.terminalBlockerIssueId ?? + issueIdForSample(sampledTerminalIdentifier); + const terminalBlockerNode = terminalBlockerIssueId + ? (nodesById.get(terminalBlockerIssueId) ?? null) + : null; + + let state: IssueBlockerAttention["state"]; + let reason: IssueBlockerAttention["reason"]; + if (attentionBlockerCount > 0) { + state = "needs_attention"; + reason = "attention_required"; + } else if (stalledBlockerCount > 0) { + state = "stalled"; + reason = "stalled_review"; + } else { + state = "covered"; + reason = topLevelEdges.every( + (edge) => nodesById.get(edge.blockerIssueId)?.parentId === root.id, + ) + ? "active_child" + : "active_dependency"; + } + + attentionMap.set( + root.id, + createIssueBlockerAttention({ + state, + reason, + unresolvedBlockerCount: topLevelEdges.length, + coveredBlockerCount, + stalledBlockerCount, + attentionBlockerCount, + pendingFinalizeBlockerIssueIds: topLevelEdges + .map((edge) => edge.blockerIssueId) + .filter((blockerIssueId) => + pendingFinalizeBlockerIssueIds.has(blockerIssueId), + ), + sampleBlockerIdentifier: + sampleEntry?.result.sampleBlockerIdentifier ?? + blockerSampleIdentifier(sampleNode), + sampleStalledBlockerIdentifier: + stalledEntry?.result.sampleStalledBlockerIdentifier ?? + sampleStalledFromChain ?? + null, + blockingTreeLive: topLevelEdges.some((edge) => + pathHasLiveWork(edge.blockerIssueId, new Set([root.id])), + ), + directBlockerIssueId: sampleEntry?.edge.blockerIssueId ?? null, + terminalBlockerIssueId, + terminalBlocker: terminalBlockerNode + ? { + id: terminalBlockerNode.id, + identifier: terminalBlockerNode.identifier, + title: terminalBlockerNode.title, + } + : null, + }), + ); + } + + return attentionMap; +} + +type IssueReviewAttentionInput = Pick; + +function reviewPathLabel( + kind: IssueReviewAttentionPath["kind"], + detail?: string | null, +) { + switch (kind) { + case "execution_participant": + return "Execution review participant"; + case "interaction": + return detail + ? `Pending ${detail.replaceAll("_", " ")}` + : "Pending issue interaction"; + case "approval": + return "Linked approval"; + case "monitor": + return "Scheduled review monitor"; + case "external_response": + return "Waiting for an external update"; + case "human_reviewer": + return "Human reviewer"; + case "active_run": + return "Active review run"; + case "queued_wake": + return detail + ? `Queued ${detail.replaceAll("_", " ")} wake` + : "Queued review wake"; + case "recovery": + return "Open review recovery"; + } +} + +function reviewAttentionNone(): IssueReviewAttention { + return { state: "none", paths: [], reason: null }; +} + +async function listIssueReviewAttentionMap( + dbOrTx: any, + companyId: string, + issueRows: IssueReviewAttentionInput[], +): Promise> { + const result = new Map(); + for (const row of issueRows) result.set(row.id, reviewAttentionNone()); + + const reviewIds = issueRows + .filter((row) => row.companyId === companyId && row.status === "in_review") + .map((row) => row.id); + if (reviewIds.length === 0) return result; + + const reviewIssues: IssueRow[] = []; + for (const chunk of chunkList( + reviewIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + reviewIssues.push( + ...(await dbOrTx + .select() + .from(issues) + .where( + and(eq(issues.companyId, companyId), inArray(issues.id, chunk), nonIdleSlackIssueCondition()), + )), + ); + } + if (reviewIssues.length === 0) return result; + + const [ + agentRows, + activeRunRows, + wakeRows, + interactionRows, + approvalRows, + recoveryActionRows, + recoveryIssueRows, + ] = await Promise.all([ + dbOrTx + .select({ + id: agents.id, + companyId: agents.companyId, + name: agents.name, + role: agents.role, + title: agents.title, + status: agents.status, + reportsTo: agents.reportsTo, + }) + .from(agents) + .where(eq(agents.companyId, companyId)), + dbOrTx + .select({ + id: heartbeatRuns.id, + companyId: heartbeatRuns.companyId, + issueId: sql< + string | null + >`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId')`, + agentId: heartbeatRuns.agentId, + status: heartbeatRuns.status, + createdAt: heartbeatRuns.createdAt, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.status, ["queued", "running"]), + inArray( + sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId')`, + reviewIds, + ), + ), + ), + dbOrTx + .select({ + id: agentWakeupRequests.id, + companyId: agentWakeupRequests.companyId, + issueId: sql`coalesce( + ${agentWakeupRequests.payload} ->> 'issueId', + ${agentWakeupRequests.payload} ->> 'taskId', + ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'issueId', + ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'taskId' + )`, + agentId: agentWakeupRequests.agentId, + status: agentWakeupRequests.status, + reason: agentWakeupRequests.reason, + createdAt: agentWakeupRequests.requestedAt, + }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + "claimed", + ]), + inArray( + sql`coalesce( + ${agentWakeupRequests.payload} ->> 'issueId', + ${agentWakeupRequests.payload} ->> 'taskId', + ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'issueId', + ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'taskId' + )`, + reviewIds, + ), + ), + ), + dbOrTx + .select({ + id: issueThreadInteractions.id, + companyId: issueThreadInteractions.companyId, + issueId: issueThreadInteractions.issueId, + status: issueThreadInteractions.status, + kind: issueThreadInteractions.kind, + createdByAgentId: issueThreadInteractions.createdByAgentId, + sourceRunId: issueThreadInteractions.sourceRunId, + addresseeAgentId: issueThreadInteractions.addresseeAgentId, + effectiveResolverPolicy: + issueThreadInteractions.effectiveResolverPolicy, + resolverPolicyProvenance: + issueThreadInteractions.resolverPolicyProvenance, + createdAt: issueThreadInteractions.createdAt, + }) + .from(issueThreadInteractions) + .where( + and( + eq(issueThreadInteractions.companyId, companyId), + eq(issueThreadInteractions.status, "pending"), + inArray(issueThreadInteractions.issueId, reviewIds), + ), + ), + dbOrTx + .select({ + id: approvals.id, + companyId: issueApprovals.companyId, + issueId: issueApprovals.issueId, + status: approvals.status, + createdAt: approvals.createdAt, + }) + .from(issueApprovals) + .innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id)) + .where( + and( + eq(issueApprovals.companyId, companyId), + eq(approvals.companyId, companyId), + inArray(approvals.status, ["pending", "revision_requested"]), + inArray(issueApprovals.issueId, reviewIds), + ), + ), + dbOrTx + .select({ + id: issueRecoveryActions.id, + companyId: issueRecoveryActions.companyId, + issueId: issueRecoveryActions.sourceIssueId, + status: issueRecoveryActions.status, + createdAt: issueRecoveryActions.createdAt, + }) + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, companyId), + inArray(issueRecoveryActions.status, ["active", "escalated"]), + inArray(issueRecoveryActions.sourceIssueId, reviewIds), + ), + ), + dbOrTx + .select({ + id: issues.id, + companyId: issues.companyId, + originKind: issues.originKind, + originId: issues.originId, + status: issues.status, + createdAt: issues.createdAt, + }) + .from(issues) + .where( + and( + eq(issues.companyId, companyId), + inArray(issues.originKind, [ + RECOVERY_ORIGIN_KINDS.strandedIssueRecovery, + RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation, + ]), + visibleIssueCondition(), + notInArray(issues.status, ["done", "cancelled"]), + ), + ), + ]); + + const recoveryPaths = [ + ...(recoveryActionRows as Array<{ + id: string; + companyId: string; + issueId: string; + status: string; + createdAt: Date; + }>), + ]; + for (const recovery of recoveryIssueRows as Array<{ + id: string; + companyId: string; + originKind: string; + originId: string | null; + status: string; + createdAt: Date; + }>) { + if ( + recovery.originKind === RECOVERY_ORIGIN_KINDS.strandedIssueRecovery && + recovery.originId && + reviewIds.includes(recovery.originId) + ) { + recoveryPaths.push({ ...recovery, issueId: recovery.originId }); + continue; + } + const parsed = parseIssueGraphLivenessIncidentKey(recovery.originId); + if (parsed?.companyId === companyId && reviewIds.includes(parsed.issueId)) { + recoveryPaths.push({ ...recovery, issueId: parsed.issueId }); + } + } + + const livenessInput: IssueGraphLivenessInput = { + issues: reviewIssues.map((issue) => ({ + id: issue.id, + companyId: issue.companyId, + identifier: issue.identifier, + title: issue.title, + status: issue.status, + projectId: issue.projectId, + goalId: issue.goalId, + parentId: issue.parentId, + assigneeAgentId: issue.assigneeAgentId, + assigneeUserId: issue.assigneeUserId, + createdByAgentId: issue.createdByAgentId, + createdByUserId: issue.createdByUserId, + conversationAgentId: issue.conversationAgentId, + conversationUserId: issue.conversationUserId, + conversationState: issue.conversationState, + executionPolicy: issue.executionPolicy, + executionState: issue.executionState, + monitorNextCheckAt: issue.monitorNextCheckAt, + monitorAttemptCount: issue.monitorAttemptCount, + })), + relations: [], + agents: agentRows, + activeRuns: activeRunRows, + queuedWakeRequests: wakeRows, + pendingInteractions: interactionRows, + pendingApprovals: approvalRows, + externalResponseWaits: await listExternalReviewWaits(dbOrTx, companyId, reviewIds), + openRecoveryIssues: recoveryPaths, + now: new Date(), + }; + const findingsByIssueId = new Map( + classifyIssueGraphLiveness(livenessInput).map((finding) => [ + finding.issueId, + finding, + ]), + ); + const agentNameById = new Map( + (agentRows as Array<{ id: string; name: string }>).map((agent) => [ + agent.id, + agent.name, + ]), + ); + const userIds = new Set(); + for (const issue of reviewIssues) { + if (issue.assigneeUserId) userIds.add(issue.assigneeUserId); + const participant = parseObject(issue.executionState).currentParticipant; + if ( + participant && + typeof participant === "object" && + !Array.isArray(participant) + ) { + const userId = (participant as Record).userId; + if (typeof userId === "string") userIds.add(userId); + } + } + const userRows = + userIds.size > 0 + ? await dbOrTx + .select({ id: authUsers.id, name: authUsers.name }) + .from(authUsers) + .where(inArray(authUsers.id, [...userIds])) + : []; + const userNameById = new Map( + (userRows as Array<{ id: string; name: string }>).map((user) => [ + user.id, + user.name, + ]), + ); + const interactionKindById = new Map( + (interactionRows as Array<{ id: string; kind: string }>).map((row) => [ + row.id, + row.kind, + ]), + ); + const interactionAudienceById = new Map( + ( + interactionRows as Array<{ + id: string; + createdByAgentId: string | null; + sourceRunId: string | null; + addresseeAgentId: string | null; + effectiveResolverPolicy: string; + resolverPolicyProvenance: string | null; + }> + ).map((row) => [row.id, row]), + ); + const wakeReasonById = new Map( + (wakeRows as Array<{ id: string; reason: string | null }>).map((row) => [ + row.id, + row.reason, + ]), + ); + + for (const issue of reviewIssues) { + const pathFacts = classifyIssueReviewPaths( + livenessInput, + livenessInput.issues.find((entry) => entry.id === issue.id)!, + ); + const paths: IssueReviewAttentionPath[] = pathFacts.map((path) => { + const interactionAudience = + path.kind === "interaction" && path.ref + ? (interactionAudienceById.get(path.ref) ?? null) + : null; + const candidateAgentId = + interactionAudience?.addresseeAgentId ?? issue.assigneeAgentId; + const interactionResponderAgentId = + interactionAudience && + candidateAgentId && + issueThreadInteractionAttentionAgentAllowed({ + agentId: candidateAgentId, + interaction: interactionAudience, + }) + ? candidateAgentId + : null; + return { + kind: path.kind, + label: reviewPathLabel( + path.kind, + path.kind === "interaction" && path.ref + ? (interactionKindById.get(path.ref) ?? null) + : path.kind === "queued_wake" && path.ref + ? (wakeReasonById.get(path.ref) ?? null) + : null, + ), + responder: path.agentId + ? (agentNameById.get(path.agentId) ?? path.agentId) + : path.userId + ? (userNameById.get(path.userId) ?? path.userId) + : path.kind === "interaction" && interactionResponderAgentId + ? (agentNameById.get(interactionResponderAgentId) ?? + interactionResponderAgentId) + : path.kind === "interaction" || path.kind === "approval" + ? "Board" + : null, + since: path.since + ? (path.since instanceof Date + ? path.since + : new Date(path.since) + ).toISOString() + : issue.updatedAt.toISOString(), + ref: path.ref, + }; + }); + + if (paths.length > 0) { + result.set(issue.id, { + state: "covered", + paths, + reason: + paths.length === 1 + ? "Review has a maintained action path." + : `Review has ${paths.length} maintained action paths.`, + }); + continue; + } + + const finding = findingsByIssueId.get(issue.id); + result.set(issue.id, { + state: "stalled", + paths: [], + reason: + finding?.reason ?? + "Issue is in review without a maintained action path.", + }); + } + + return result; +} + +const issueListSelect = { + externalConversationState: externalConversationStateSql(), + conversationAgentId: issues.conversationAgentId, + conversationUserId: issues.conversationUserId, + conversationState: issues.conversationState, + conversationSessionGeneration: issues.conversationSessionGeneration, + conversationBoundaryCommentId: issues.conversationBoundaryCommentId, + id: issues.id, + companyId: issues.companyId, + projectId: issues.projectId, + projectWorkspaceId: issues.projectWorkspaceId, + goalId: issues.goalId, + parentId: issues.parentId, + visibility: issues.visibility, + privacyRootIssueId: issues.privacyRootIssueId, + privacyParentIssueId: issues.privacyParentIssueId, + title: issues.title, + titleNeedsGeneration: issues.titleNeedsGeneration, + description: sql` + CASE + WHEN ${issues.description} IS NULL THEN NULL + ELSE encode( + substring( + convert_to(${issues.description}, current_setting('server_encoding')) + FROM 1 FOR ${ISSUE_LIST_DESCRIPTION_MAX_BYTES} + ), + 'base64' + ) + END + `, + descriptionTruncated: sql` + CASE + WHEN ${issues.description} IS NULL THEN false + ELSE length(${issues.description}) > ${ISSUE_LIST_DESCRIPTION_MAX_CHARS} + END + `, + status: issues.status, + statusVersion: issues.statusVersion, + lastStatusDecisionId: issues.lastStatusDecisionId, + workMode: issues.workMode, + harnessKind: issues.harnessKind, + priority: issues.priority, + reviewPolicy: issues.reviewPolicy, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + executionAgentNameKey: issues.executionAgentNameKey, + executionLockedAt: issues.executionLockedAt, + createdByAgentId: issues.createdByAgentId, + createdByUserId: issues.createdByUserId, + responsibleUserId: issues.responsibleUserId, + issueNumber: issues.issueNumber, + identifier: issues.identifier, + originKind: issues.originKind, + originId: issues.originId, + originRunId: issues.originRunId, + originIdentityContextId: issues.originIdentityContextId, + continuationIdentityContextId: issues.continuationIdentityContextId, + originFingerprint: issues.originFingerprint, + requestDepth: issues.requestDepth, + billingCode: issues.billingCode, + assigneeAdapterOverrides: issues.assigneeAdapterOverrides, + executionPolicy: sql`null`, + executionState: sql`null`, + monitorNextCheckAt: issues.monitorNextCheckAt, + monitorWakeRequestedAt: issues.monitorWakeRequestedAt, + monitorLastTriggeredAt: issues.monitorLastTriggeredAt, + monitorAttemptCount: issues.monitorAttemptCount, + monitorNotes: issues.monitorNotes, + monitorScheduledBy: issues.monitorScheduledBy, + executionWorkspaceId: issues.executionWorkspaceId, + executionWorkspacePreference: issues.executionWorkspacePreference, + executionWorkspaceSettings: sql`null`, + workspaceSelection: sql`null`, + workspacePendingSelection: sql`null`, + workspaceBindingRevision: issues.workspaceBindingRevision, + sourceTrust: issues.sourceTrust, + unblockDescriptor: issues.unblockDescriptor, + blockedTransitionAt: issues.blockedTransitionAt, + blockedOwnerNotifiedAt: issues.blockedOwnerNotifiedAt, + startedAt: issues.startedAt, + completedAt: issues.completedAt, + cancelledAt: issues.cancelledAt, + hiddenAt: issues.hiddenAt, + createdAt: issues.createdAt, + updatedAt: issues.updatedAt, +}; + +function withActiveRuns( + issueRows: IssueWithLabels[], + runMap: Map, +): IssueWithLabelsAndRun[] { + return issueRows.map((row) => ({ + ...row, + activeRun: row.executionRunId + ? (runMap.get(row.executionRunId) ?? null) + : null, + })); +} + +async function userCommentStatsForIssues( + dbOrTx: any, + companyId: string, + userId: string, + issueIds: string[], +): Promise { + const stats: IssueUserCommentStats[] = []; + for (const issueIdChunk of chunkList( + issueIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const rows = await dbOrTx + .select({ + issueId: issueComments.issueId, + myLastCommentAt: sql` + MAX(CASE WHEN ${issueComments.authorUserId} = ${userId} AND ${directHumanCommentCondition()} + THEN ${issueComments.createdAt} END) + `, + lastExternalCommentAt: sql` + MAX( + CASE + WHEN ${issueComments.authorUserId} IS NULL OR ${issueComments.authorUserId} <> ${userId} + THEN ${issueComments.createdAt} + END + ) + `, + }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, companyId), + inArray(issueComments.issueId, issueIdChunk), + ), + ) + .groupBy(issueComments.issueId); + const unreadRows = await dbOrTx.select({ id: issues.id }).from(issues).where(and( + eq(issues.companyId, companyId), + inArray(issues.id, issueIdChunk), + unreadForUserCondition(companyId, userId), + )); + const unreadIds = new Set(unreadRows.map((row: { id: string }) => row.id)); + const commentsByIssueId = new Map( + rows.map((row: { issueId: string; myLastCommentAt: Date | null; lastExternalCommentAt: Date | null }) => [row.issueId, row]), + ); + for (const issueId of issueIdChunk) stats.push({ + issueId, + myLastCommentAt: commentsByIssueId.get(issueId)?.myLastCommentAt ?? null, + lastExternalCommentAt: commentsByIssueId.get(issueId)?.lastExternalCommentAt ?? null, + isUnreadForMe: unreadIds.has(issueId), + }); + } + return stats; +} + +async function userReadStatsForIssues( + dbOrTx: any, + companyId: string, + userId: string, + issueIds: string[], +): Promise { + const stats: IssueReadStat[] = []; + for (const issueIdChunk of chunkList( + issueIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const rows = await dbOrTx + .select({ + issueId: issueReadStates.issueId, + myLastReadAt: issueReadStates.lastReadAt, + }) + .from(issueReadStates) + .where( + and( + eq(issueReadStates.companyId, companyId), + eq(issueReadStates.userId, userId), + inArray(issueReadStates.issueId, issueIdChunk), + ), + ); + stats.push(...rows); + } + return stats; +} + +async function lastActivityStatsForIssues( + dbOrTx: any, + companyId: string, + issueIds: string[], +): Promise { + const byIssueId = new Map(); + for (const issueIdChunk of chunkList( + issueIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const [commentRows, logRows] = await Promise.all([ + dbOrTx + .select({ + issueId: issueComments.issueId, + latestCommentAt: sql`MAX(${issueComments.createdAt})`, + }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, companyId), + inArray(issueComments.issueId, issueIdChunk), + ), + ) + .groupBy(issueComments.issueId), + dbOrTx + .select({ + issueId: activityLog.entityId, + latestLogAt: sql`MAX(${activityLog.createdAt})`, + }) + .from(activityLog) + .where( + and( + eq(activityLog.companyId, companyId), + eq(activityLog.entityType, "issue"), + inArray(activityLog.entityId, issueIdChunk), + sql`NOT ${automaticReadRecoveryCleanupCondition()}`, + sql`${activityLog.action} NOT IN (${sql.join( + ISSUE_LOCAL_INBOX_ACTIVITY_ACTIONS.map( + (action) => sql`${action}`, + ), + sql`, `, + )})`, + ), + ) + .groupBy(activityLog.entityId), + ]); + + for (const row of commentRows) { + byIssueId.set(row.issueId, { + issueId: row.issueId, + latestCommentAt: row.latestCommentAt, + latestLogAt: null, + }); + } + for (const row of logRows) { + const existing = byIssueId.get(row.issueId); + if (existing) existing.latestLogAt = row.latestLogAt; + else { + byIssueId.set(row.issueId, { + issueId: row.issueId, + latestCommentAt: null, + latestLogAt: row.latestLogAt, + }); + } + } + } + return [...byIssueId.values()]; +} + +async function blockedByMapForIssues( + dbOrTx: any, + companyId: string, + issueIds: string[], +): Promise> { + const map = new Map(); + const uniqueIssueIds = [...new Set(issueIds)]; + if (uniqueIssueIds.length === 0) return map; + + for (const issueId of uniqueIssueIds) { + map.set(issueId, []); + } + + for (const issueIdChunk of chunkList( + uniqueIssueIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const rows = await dbOrTx + .select({ + currentIssueId: issueRelations.relatedIssueId, + relatedId: issues.id, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + priority: issues.priority, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.type, "blocks"), + inArray(issueRelations.relatedIssueId, issueIdChunk), + ), + ); + + for (const row of rows) { + const blockedBy = map.get(row.currentIssueId); + if (!blockedBy) continue; + blockedBy.push({ + id: row.relatedId, + identifier: row.identifier, + title: row.title, + status: row.status as IssueRelationIssueSummary["status"], + priority: row.priority as IssueRelationIssueSummary["priority"], + assigneeAgentId: row.assigneeAgentId, + assigneeUserId: row.assigneeUserId, + }); + } + } + + for (const blockedBy of map.values()) { + blockedBy.sort((a, b) => a.title.localeCompare(b.title)); + } + + return map; +} + +const BLOCKED_INBOX_TERMINAL_STATUSES = ["done", "cancelled"] as const; +const BLOCKED_INBOX_ACTIVE_RUN_STATUSES = ["queued", "running"] as const; +const BLOCKED_INBOX_ACTIVE_WAKE_STATUSES = + SUCCESSFUL_RUN_HANDOFF_LIVE_WAKE_STATUSES; +const BLOCKED_INBOX_PENDING_INTERACTION_STATUSES = ["pending"] as const; +const BLOCKED_INBOX_PENDING_APPROVAL_STATUSES = [ + "pending", + "revision_requested", +] as const; +const BLOCKED_INBOX_RECOVERY_ORIGIN_KINDS = [ + "harness_liveness_escalation", + "stranded_issue_recovery", +] as const; +const BLOCKED_INBOX_SUCCESSFUL_RUN_HANDOFF_ACTIONS = [ + "issue.successful_run_handoff_required", + "issue.successful_run_handoff_resolved", + "issue.successful_run_handoff_escalated", +] as const; + +type BlockedInboxIssueRow = IssueRow & { + labels?: IssueLabelRow[]; + labelIds?: string[]; +}; +type BlockedInboxInteractionRow = { + id: string; + issueId: string; + kind: string; + createdAt: Date; +}; +type BlockedInboxApprovalRow = { + approvalId: string; + issueId: string; + createdAt: Date; +}; + +function issueRef( + row: + | Pick< + IssueRow, + | "id" + | "identifier" + | "title" + | "status" + | "priority" + | "assigneeAgentId" + | "assigneeUserId" + > + | null + | undefined, +): IssueBlockedInboxIssueRef | null { + if (!row) return null; + return { + id: row.id, + identifier: row.identifier, + title: row.title, + status: row.status as IssueBlockedInboxIssueRef["status"], + priority: row.priority as IssueBlockedInboxIssueRef["priority"], + assigneeAgentId: row.assigneeAgentId, + assigneeUserId: row.assigneeUserId, + }; +} + +function hasPlanDocumentCondition( + companyId: string, + hasPlanDocument: boolean, +): SQL { + const existsPlanDocument = sql` + EXISTS ( + SELECT 1 + FROM ${issueDocuments} + WHERE ${issueDocuments.companyId} = ${companyId} + AND ${issueDocuments.issueId} = ${issues.id} + AND ${issueDocuments.key} = 'plan' + ) + `; + return hasPlanDocument + ? existsPlanDocument + : sql`NOT ${existsPlanDocument}`; +} + +function isoDate(value: Date | string | null | undefined): string | null { + if (!value) return null; + const date = value instanceof Date ? value : new Date(value); + return Number.isNaN(date.getTime()) ? null : date.toISOString(); +} + +function attentionBase(input: { + state: IssueBlockedInboxAttention["state"]; + reason: IssueBlockedInboxAttention["reason"]; + severity: IssueBlockedInboxAttention["severity"]; + stoppedSinceAt: Date | string | null | undefined; + owner: IssueBlockedInboxAttention["owner"]; + action: IssueBlockedInboxAttention["action"]; + sourceIssue: IssueBlockedInboxIssueRef | null; + leafIssue?: IssueBlockedInboxIssueRef | null; + recoveryIssue?: IssueBlockedInboxIssueRef | null; + approvalId?: string | null; + interactionId?: string | null; + sampleIssueIdentifier?: string | null; + externalDetailsRedacted?: boolean; +}): IssueBlockedInboxAttention { + return { + kind: "blocked", + state: input.state, + reason: input.reason, + severity: input.severity, + stoppedSinceAt: isoDate(input.stoppedSinceAt), + owner: input.owner, + action: input.action, + sourceIssue: input.sourceIssue, + leafIssue: input.leafIssue ?? null, + recoveryIssue: input.recoveryIssue ?? null, + approvalId: input.approvalId ?? null, + interactionId: input.interactionId ?? null, + sampleIssueIdentifier: + input.sampleIssueIdentifier ?? + input.leafIssue?.identifier ?? + input.recoveryIssue?.identifier ?? + input.sourceIssue?.identifier ?? + null, + redaction: { + externalDetailsRedacted: input.externalDetailsRedacted ?? false, + secretFieldsOmitted: true, + }, + }; +} + +function readSuccessfulRunHandoffFromActivity(row: { + action: string; + agentId: string | null; + runId: string | null; + details: Record | null; + createdAt: Date; +}): SuccessfulRunHandoffState | null { + const details = row.details ?? {}; + const state = + row.action === "issue.successful_run_handoff_required" + ? "required" + : row.action === "issue.successful_run_handoff_resolved" + ? "resolved" + : row.action === "issue.successful_run_handoff_escalated" + ? "escalated" + : null; + if (!state) return null; + + const detectedProgressSummary = + readStringFromRecord(details, "detectedProgressSummary") ?? + readStringFromRecord(details, "detected_progress_summary") ?? + null; + + return { + state, + required: state === "required", + hasLiveContinuation: false, + sourceRunId: + readStringFromRecord(details, "sourceRunId") ?? + readStringFromRecord(details, "source_run_id") ?? + readStringFromRecord(details, "resumeFromRunId") ?? + row.runId ?? + null, + correctiveRunId: + readStringFromRecord(details, "correctiveRunId") ?? + readStringFromRecord(details, "corrective_run_id") ?? + (state !== "required" ? row.runId : null), + assigneeAgentId: + readStringFromRecord(details, "assigneeAgentId") ?? + readStringFromRecord(details, "agentId") ?? + row.agentId ?? + null, + detectedProgressSummary: detectedProgressSummary + ? redactSensitiveText(detectedProgressSummary) + : null, + createdAt: row.createdAt, + }; +} + +async function listSuccessfulRunHandoffMapForIssues( + dbOrTx: any, + companyId: string, + issueIds: string[], + options?: { hydrateLiveness?: boolean }, +): Promise> { + const uniqueIssueIds = [...new Set(issueIds)]; + const states = new Map(); + if (uniqueIssueIds.length === 0) return states; + + for (const issueIdChunk of chunkList( + uniqueIssueIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const rows = await dbOrTx + .select({ + entityId: activityLog.entityId, + action: activityLog.action, + agentId: activityLog.agentId, + runId: activityLog.runId, + details: activityLog.details, + createdAt: activityLog.createdAt, + }) + .from(activityLog) + .where( + and( + eq(activityLog.companyId, companyId), + eq(activityLog.entityType, "issue"), + inArray(activityLog.entityId, issueIdChunk), + inArray(activityLog.action, [ + ...BLOCKED_INBOX_SUCCESSFUL_RUN_HANDOFF_ACTIONS, + ]), + ), + ) + .orderBy( + activityLog.entityId, + desc(activityLog.createdAt), + desc(activityLog.id), + ); + + for (const row of rows as Array<{ + entityId: string; + action: string; + agentId: string | null; + runId: string | null; + details: Record | null; + createdAt: Date; + }>) { + if (states.has(row.entityId)) continue; + const state = readSuccessfulRunHandoffFromActivity(row); + if (state) states.set(row.entityId, state); + } + } + + return options?.hydrateLiveness === false + ? states + : hydrateSuccessfulRunHandoffLiveness(dbOrTx, companyId, states); +} + +function externalWaitFromDescription( + description: string | null, +): { owner: string; action: string } | null { + if (!description) return null; + const owner = description + .match(/^\s*external owner\s*:\s*(.+)$/im)?.[1] + ?.trim(); + const action = description + .match(/^\s*external action\s*:\s*(.+)$/im)?.[1] + ?.trim(); + if (!owner || !action) return null; + return { + owner: owner.slice(0, 120), + action: action.slice(0, 240), + }; +} + +function escapeRegExp(value: string) { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function redactExternalWaitDescription( + description: string | null | undefined, + external: { owner: string; action: string } | null, +) { + if (!description) return null; + let redacted = description + .split(/\r?\n/) + .filter((line) => !/^\s*external\s+(?:owner|action)\s*:/i.test(line)) + .join("\n"); + + for (const value of [external?.owner, external?.action]) { + if (!value) continue; + redacted = redacted.replace( + new RegExp(escapeRegExp(value), "gi"), + "[redacted external wait detail]", + ); + } + + redacted = redacted.replace(/\n{3,}/g, "\n\n").trim(); + return redacted.length > 0 ? redacted : null; +} + +function blockedInboxResponseDescription( + attention: IssueBlockedInboxAttention, + row: BlockedInboxIssueRow, +) { + if (!attention.redaction.externalDetailsRedacted) return row.description; + return redactExternalWaitDescription( + row.description, + externalWaitFromDescription(row.description), + ); +} + +function blockedInboxSearchText( + attention: IssueBlockedInboxAttention, + row: BlockedInboxIssueRow, +) { + return [ + row.identifier, + row.title, + blockedInboxResponseDescription(attention, row), + attention.sourceIssue?.identifier, + attention.sourceIssue?.title, + attention.leafIssue?.identifier, + attention.leafIssue?.title, + attention.recoveryIssue?.identifier, + attention.recoveryIssue?.title, + attention.action.label, + attention.action.detail, + ] + .filter( + (value): value is string => typeof value === "string" && value.length > 0, + ) + .join(" ") + .toLowerCase(); +} + +function blockedInboxSeverityRank( + severity: IssueBlockedInboxAttention["severity"], +) { + switch (severity) { + case "critical": + return 0; + case "high": + return 1; + case "medium": + return 2; + case "low": + return 3; + } +} + +function issuePriorityRank(priority: string) { + switch (priority) { + case "critical": + return 0; + case "high": + return 1; + case "medium": + return 2; + case "low": + return 3; + default: + return 4; + } +} + +function compareBlockedInboxRows( + left: BlockedInboxIssueRow & { + blockedInboxAttention: IssueBlockedInboxAttention; + lastActivityAt?: Date | null; + }, + right: BlockedInboxIssueRow & { + blockedInboxAttention: IssueBlockedInboxAttention; + lastActivityAt?: Date | null; + }, +) { + const leftAttention = left.blockedInboxAttention; + const rightAttention = right.blockedInboxAttention; + const severity = + blockedInboxSeverityRank(leftAttention.severity) - + blockedInboxSeverityRank(rightAttention.severity); + if (severity !== 0) return severity; + + const leftStopped = leftAttention.stoppedSinceAt + ? new Date(leftAttention.stoppedSinceAt).getTime() + : Number.POSITIVE_INFINITY; + const rightStopped = rightAttention.stoppedSinceAt + ? new Date(rightAttention.stoppedSinceAt).getTime() + : Number.POSITIVE_INFINITY; + if (leftStopped !== rightStopped) return leftStopped - rightStopped; + + const priority = + issuePriorityRank(left.priority) - issuePriorityRank(right.priority); + if (priority !== 0) return priority; + + const leftActivity = left.lastActivityAt + ? new Date(left.lastActivityAt).getTime() + : new Date(left.updatedAt).getTime(); + const rightActivity = right.lastActivityAt + ? new Date(right.lastActivityAt).getTime() + : new Date(right.updatedAt).getTime(); + if (leftActivity !== rightActivity) return rightActivity - leftActivity; + + return right.id.localeCompare(left.id); +} + +async function listIssueBlockedInboxAttentionMap( + dbOrTx: any, + companyId: string, + issueRows: BlockedInboxIssueRow[], +): Promise> { + const rowIssueIds = [...new Set(issueRows.map((row) => row.id))]; + const result = new Map(); + if (rowIssueIds.length === 0) return result; + + const [graphIssueRows, graphRelationRows, companyAgentRows] = + await Promise.all([ + dbOrTx + .select() + .from(issues) + .where( + and( + eq(issues.companyId, companyId), + visibleIssueCondition(), + ne(issues.status, "done"), + ), + ), + dbOrTx + .select({ + companyId: issueRelations.companyId, + blockerIssueId: issueRelations.issueId, + blockedIssueId: issueRelations.relatedIssueId, + }) + .from(issueRelations) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.type, "blocks"), + ), + ), + dbOrTx + .select({ + id: agents.id, + companyId: agents.companyId, + name: agents.name, + role: agents.role, + title: agents.title, + status: agents.status, + reportsTo: agents.reportsTo, + }) + .from(agents) + .where(eq(agents.companyId, companyId)), + ]); + + const graphIssues = graphIssueRows as IssueRow[]; + const graphRelations = graphRelationRows as Array<{ + companyId: string; + blockerIssueId: string; + blockedIssueId: string; + }>; + const companyAgents = companyAgentRows as Array<{ + id: string; + companyId: string; + name: string; + role: string; + title: string | null; + status: string; + reportsTo: string | null; + }>; + const graphIssueIds = graphIssues.map((issue) => issue.id); + const issuesById = new Map( + graphIssues.map((issue) => [issue.id, issue]), + ); + + const [ + activeRunRows, + wakeRows, + scheduledRetryRows, + interactionRows, + approvalRows, + handoffMap, + ] = await Promise.all([ + graphIssueIds.length === 0 + ? Promise.resolve([]) + : dbOrTx + .select({ + companyId: heartbeatRuns.companyId, + issueId: sql`coalesce( + ${heartbeatRuns.contextSnapshot} ->> 'issueId', + ${heartbeatRuns.contextSnapshot} ->> 'taskId' + )`, + agentId: heartbeatRuns.agentId, + status: heartbeatRuns.status, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.status, [ + ...BLOCKED_INBOX_ACTIVE_RUN_STATUSES, + ]), + inArray( + sql`coalesce( + ${heartbeatRuns.contextSnapshot} ->> 'issueId', + ${heartbeatRuns.contextSnapshot} ->> 'taskId' + )`, + graphIssueIds, + ), + ), + ), + graphIssueIds.length === 0 + ? Promise.resolve([]) + : dbOrTx + .select({ + companyId: agentWakeupRequests.companyId, + issueId: sql`coalesce( + ${agentWakeupRequests.payload} ->> 'issueId', + ${agentWakeupRequests.payload} ->> 'taskId', + ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'issueId', + ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'taskId' + )`, + agentId: agentWakeupRequests.agentId, + status: agentWakeupRequests.status, + }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + inArray(agentWakeupRequests.status, [ + ...BLOCKED_INBOX_ACTIVE_WAKE_STATUSES, + ]), + inArray( + sql`coalesce( + ${agentWakeupRequests.payload} ->> 'issueId', + ${agentWakeupRequests.payload} ->> 'taskId', + ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'issueId', + ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'taskId' + )`, + graphIssueIds, + ), + ), + ), + graphIssueIds.length === 0 + ? Promise.resolve([]) + : dbOrTx + .select({ + companyId: heartbeatRuns.companyId, + issueId: sql`coalesce( + ${heartbeatRuns.contextSnapshot} ->> 'issueId', + ${heartbeatRuns.contextSnapshot} ->> 'taskId' + )`, + agentId: heartbeatRuns.agentId, + status: heartbeatRuns.status, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.status, "scheduled_retry"), + inArray( + sql`coalesce( + ${heartbeatRuns.contextSnapshot} ->> 'issueId', + ${heartbeatRuns.contextSnapshot} ->> 'taskId' + )`, + graphIssueIds, + ), + ), + ), + graphIssueIds.length === 0 + ? Promise.resolve([]) + : dbOrTx + .select({ + id: issueThreadInteractions.id, + issueId: issueThreadInteractions.issueId, + kind: issueThreadInteractions.kind, + createdAt: issueThreadInteractions.createdAt, + }) + .from(issueThreadInteractions) + .where( + and( + eq(issueThreadInteractions.companyId, companyId), + inArray(issueThreadInteractions.status, [ + ...BLOCKED_INBOX_PENDING_INTERACTION_STATUSES, + ]), + inArray(issueThreadInteractions.issueId, graphIssueIds), + ), + ), + graphIssueIds.length === 0 + ? Promise.resolve([]) + : dbOrTx + .select({ + approvalId: approvals.id, + issueId: issueApprovals.issueId, + createdAt: approvals.createdAt, + }) + .from(issueApprovals) + .innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id)) + .where( + and( + eq(issueApprovals.companyId, companyId), + eq(approvals.companyId, companyId), + inArray(approvals.status, [ + ...BLOCKED_INBOX_PENDING_APPROVAL_STATUSES, + ]), + inArray(issueApprovals.issueId, graphIssueIds), + ), + ), + listSuccessfulRunHandoffMapForIssues(dbOrTx, companyId, rowIssueIds, { + hydrateLiveness: false, + }), + ]); + + const pendingInteractions = ( + interactionRows as BlockedInboxInteractionRow[] + ).map((row) => ({ + companyId, + issueId: row.issueId, + status: "pending", + })); + const pendingApprovals = (approvalRows as BlockedInboxApprovalRow[]).map( + (row) => ({ + companyId, + issueId: row.issueId, + status: "pending", + }), + ); + + const openRecoveryIssues = graphIssues + .filter((issue) => + BLOCKED_INBOX_RECOVERY_ORIGIN_KINDS.includes( + issue.originKind as (typeof BLOCKED_INBOX_RECOVERY_ORIGIN_KINDS)[number], + ), + ) + .filter( + (issue) => + !BLOCKED_INBOX_TERMINAL_STATUSES.includes( + issue.status as (typeof BLOCKED_INBOX_TERMINAL_STATUSES)[number], + ), + ) + .flatMap((issue) => { + const entries = [{ companyId, issueId: issue.id, status: issue.status }]; + if (issue.originKind === "harness_liveness_escalation") { + const parsed = parseIssueGraphLivenessIncidentKey(issue.originId); + if (parsed?.companyId === companyId) { + entries.push({ + companyId, + issueId: parsed.issueId, + status: issue.status, + }); + entries.push({ + companyId, + issueId: parsed.leafIssueId, + status: issue.status, + }); + } + } else if ( + issue.originKind === "stranded_issue_recovery" && + issue.originId + ) { + entries.push({ + companyId, + issueId: issue.originId, + status: issue.status, + }); + } + return entries; + }); + + const findings = classifyIssueGraphLiveness({ + externalResponseWaits: await listExternalReviewWaits(dbOrTx, companyId, graphIssues.filter(issue => issue.status === "in_review").map(issue => issue.id)), + issues: graphIssues.map((issue) => ({ + id: issue.id, + companyId: issue.companyId, + identifier: issue.identifier, + title: issue.title, + status: issue.status, + projectId: issue.projectId, + goalId: issue.goalId, + parentId: issue.parentId, + assigneeAgentId: issue.assigneeAgentId, + assigneeUserId: issue.assigneeUserId, + createdByAgentId: issue.createdByAgentId, + createdByUserId: issue.createdByUserId, + conversationAgentId: issue.conversationAgentId, + conversationUserId: issue.conversationUserId, + conversationState: issue.conversationState, + executionPolicy: issue.executionPolicy, + executionState: issue.executionState, + monitorNextCheckAt: issue.monitorNextCheckAt, + monitorAttemptCount: issue.monitorAttemptCount, + })), + relations: graphRelations, + agents: companyAgents, + activeRuns: ( + activeRunRows as Array<{ + companyId: string; + issueId: string | null; + agentId: string | null; + status: string; + }> + ).flatMap((row) => + row.issueId + ? [ + { + companyId: row.companyId, + issueId: row.issueId, + agentId: row.agentId, + status: row.status, + }, + ] + : [], + ), + queuedWakeRequests: [ + ...(wakeRows as Array<{ + companyId: string; + issueId: string | null; + agentId: string | null; + status: string; + }>), + ...(scheduledRetryRows as Array<{ + companyId: string; + issueId: string | null; + agentId: string | null; + status: string; + }>), + ].flatMap((row) => + row.issueId + ? [ + { + companyId: row.companyId, + issueId: row.issueId, + agentId: row.agentId, + status: row.status, + }, + ] + : [], + ), + pendingInteractions, + pendingApprovals, + openRecoveryIssues, + now: new Date(), + }); + const findingByIssueId = new Map(); + for (const finding of findings) { + if (!findingByIssueId.has(finding.issueId)) + findingByIssueId.set(finding.issueId, finding); + } + + const interactionByIssueId = new Map(); + for (const row of interactionRows as BlockedInboxInteractionRow[]) { + if (!interactionByIssueId.has(row.issueId)) + interactionByIssueId.set(row.issueId, row); + } + const approvalByIssueId = new Map(); + for (const row of approvalRows as BlockedInboxApprovalRow[]) { + if (!approvalByIssueId.has(row.issueId)) + approvalByIssueId.set(row.issueId, row); + } + const liveHandoffRunIssueIds = new Set( + [ + ...(activeRunRows as Array<{ issueId: string | null }>), + ...(scheduledRetryRows as Array<{ issueId: string | null }>), + ].flatMap((row) => (row.issueId ? [row.issueId] : [])), + ); + const liveHandoffWakeIssueIds = new Set( + (wakeRows as Array<{ issueId: string | null }>).flatMap((row) => + row.issueId ? [row.issueId] : [], + ), + ); + + for (const row of issueRows) { + if ( + row.companyId !== companyId || + BLOCKED_INBOX_TERMINAL_STATUSES.includes( + row.status as (typeof BLOCKED_INBOX_TERMINAL_STATUSES)[number], + ) || + row.hiddenAt + ) { + continue; + } + const source = issueRef(row); + const handoff = handoffMap.get(row.id); + const hasLiveHandoffContinuation = Boolean( + (handoff?.state === "required" || handoff?.state === "escalated") && + (liveHandoffRunIssueIds.has(row.id) || + liveHandoffWakeIssueIds.has(row.id)), + ); + if ( + handoff && + !hasLiveHandoffContinuation && + (handoff.required || handoff.state === "escalated") + ) { + result.set( + row.id, + attentionBase({ + state: "missing_disposition", + reason: "missing_successful_run_disposition", + severity: "high", + stoppedSinceAt: handoff.createdAt ?? row.updatedAt, + owner: { + type: row.assigneeAgentId + ? "agent" + : row.assigneeUserId + ? "user" + : "unknown", + agentId: row.assigneeAgentId, + userId: row.assigneeUserId, + label: null, + }, + action: { + label: "Choose disposition", + detail: + "Choose exactly one final disposition: done, cancelled, review/input, blocked with owner, delegated follow-up, or queued continuation.", + }, + sourceIssue: source, + }), + ); + continue; + } + + if ( + BLOCKED_INBOX_RECOVERY_ORIGIN_KINDS.includes( + row.originKind as (typeof BLOCKED_INBOX_RECOVERY_ORIGIN_KINDS)[number], + ) + ) { + let sourceIssue: IssueBlockedInboxIssueRef | null = null; + let leafIssue: IssueBlockedInboxIssueRef | null = null; + if (row.originKind === "harness_liveness_escalation") { + const parsed = parseIssueGraphLivenessIncidentKey(row.originId); + if (parsed?.companyId === companyId) { + sourceIssue = issueRef(issuesById.get(parsed.issueId)); + leafIssue = issueRef(issuesById.get(parsed.leafIssueId)); + } + } else if (row.originKind === "stranded_issue_recovery" && row.originId) { + sourceIssue = issueRef(issuesById.get(row.originId)); + } + result.set( + row.id, + attentionBase({ + state: "recovery_open", + reason: "open_recovery_issue", + severity: "high", + stoppedSinceAt: row.createdAt, + owner: { + type: row.assigneeAgentId + ? "agent" + : row.assigneeUserId + ? "user" + : "unknown", + agentId: row.assigneeAgentId, + userId: row.assigneeUserId, + label: null, + }, + action: { + label: "Resolve recovery", + detail: + "Restore a live path for the source work or record why this recovery issue is a false positive.", + }, + sourceIssue: sourceIssue ?? source, + leafIssue, + recoveryIssue: source, + }), + ); + continue; + } + + const interaction = interactionByIssueId.get(row.id); + if (interaction) { + const isUserQuestion = + interaction.kind === "ask_user_questions" && + Boolean(row.assigneeUserId); + result.set( + row.id, + attentionBase({ + state: "awaiting_decision", + reason: isUserQuestion + ? "pending_user_decision" + : "pending_board_decision", + severity: "medium", + stoppedSinceAt: interaction.createdAt, + owner: isUserQuestion + ? { + type: "user", + agentId: null, + userId: row.assigneeUserId, + label: null, + } + : { type: "board", agentId: null, userId: null, label: "Board" }, + action: { + label: isUserQuestion ? "Answer question" : "Answer confirmation", + detail: + "Respond to the pending issue-thread interaction so the assignee has a live next action.", + }, + sourceIssue: source, + interactionId: interaction.id, + }), + ); + continue; + } + + const approval = approvalByIssueId.get(row.id); + if (approval) { + result.set( + row.id, + attentionBase({ + state: "awaiting_decision", + reason: "pending_board_decision", + severity: "medium", + stoppedSinceAt: approval.createdAt, + owner: { type: "board", agentId: null, userId: null, label: "Board" }, + action: { + label: "Decide approval", + detail: + "Approve, reject, or request revision on the linked approval.", + }, + sourceIssue: source, + approvalId: approval.approvalId, + }), + ); + continue; + } + + const finding = findingByIssueId.get(row.id); + if (finding) { + const leaf = + finding.dependencyPath.length > 1 + ? issuesById.get( + finding.dependencyPath[finding.dependencyPath.length - 1]! + .issueId, + ) + : issuesById.get(finding.recoveryIssueId); + const ownerAgentId = + finding.state === "blocked_by_unassigned_issue" + ? null + : (finding.recommendedOwnerAgentId ?? + row.assigneeAgentId ?? + leaf?.assigneeAgentId ?? + null); + result.set( + row.id, + attentionBase({ + state: "needs_attention", + reason: finding.state as IssueBlockedInboxAttention["reason"], + severity: + finding.state === "blocked_by_assigned_backlog_issue" || + finding.state === "in_review_without_action_path" + ? "high" + : finding.severity === "critical" + ? "critical" + : "high", + stoppedSinceAt: leaf?.updatedAt ?? row.updatedAt, + owner: { + type: ownerAgentId + ? "agent" + : leaf?.assigneeUserId + ? "user" + : "unknown", + agentId: ownerAgentId, + userId: leaf?.assigneeUserId ?? null, + label: null, + }, + action: { + label: (() => { + switch (finding.state) { + case "blocked_by_unassigned_issue": + return "Assign blocker"; + case "blocked_by_assigned_backlog_issue": + return "Resume parked blocker"; + case "blocked_by_uninvokable_assignee": + return "Assign active owner"; + case "blocked_by_cancelled_issue": + return "Replace blocker"; + case "invalid_review_participant": + return "Repair review participant"; + case "in_review_without_action_path": + return "Choose review path"; + } + })(), + detail: finding.recommendedAction, + }, + sourceIssue: source, + leafIssue: issueRef(leaf), + recoveryIssue: issueRef(issuesById.get(finding.recoveryIssueId)), + sampleIssueIdentifier: leaf?.identifier ?? finding.identifier, + }), + ); + continue; + } + + const hasMonitor = Boolean( + row.monitorNextCheckAt && row.monitorNextCheckAt.getTime() > Date.now(), + ); + const external = + row.status === "blocked" && !hasMonitor + ? externalWaitFromDescription(row.description) + : null; + if (external) { + result.set( + row.id, + attentionBase({ + state: "external_wait", + reason: "external_owner_action", + severity: "medium", + stoppedSinceAt: row.updatedAt, + owner: { type: "external", agentId: null, userId: null, label: null }, + action: { + label: "External owner action", + detail: null, + }, + sourceIssue: source, + externalDetailsRedacted: true, + }), + ); + continue; + } + + const blockerAttention = await listIssueBlockerAttentionMap( + dbOrTx, + companyId, + [row], + ); + const blockerState = blockerAttention.get(row.id); + if ( + row.status === "blocked" && + (blockerState?.state === "needs_attention" || + blockerState?.state === "stalled") + ) { + result.set( + row.id, + attentionBase({ + state: "needs_attention", + reason: "blocked_chain_stalled", + severity: "high", + stoppedSinceAt: row.updatedAt, + owner: { type: "unknown", agentId: null, userId: null, label: null }, + action: { + label: "Inspect blocker chain", + detail: + "Inspect the stalled blocker or review leaf and make the next owner/action explicit.", + }, + sourceIssue: source, + sampleIssueIdentifier: + blockerState.sampleStalledBlockerIdentifier ?? + blockerState.sampleBlockerIdentifier, + }), + ); + } + } + + return result; +} + +function parseIssueAssigneeAgentFilter( + assigneeAgentId: IssueFilters["assigneeAgentId"], +): string | null | undefined { + const normalizedRaw = + typeof assigneeAgentId === "string" + ? assigneeAgentId.trim() + : assigneeAgentId; + const normalized = normalizedRaw === "" ? undefined : normalizedRaw; + if (typeof normalized !== "string") return normalized; + return normalized.toLowerCase() === "null" ? null : normalized; +} + +function assertValidAssigneeAgentFilter( + assigneeAgentFilter: string | null | undefined, +) { + if ( + typeof assigneeAgentFilter === "string" && + !isUuidLike(assigneeAgentFilter) + ) { + throw unprocessable("assigneeAgentId must be a UUID or 'null'"); + } +} + +function creationVisibilityCondition(companyId: string, filters?: IssueFilters) { + if (filters?.includeAgentCreatedTasks !== false) return undefined; + return filters.creationVisibilityUserId + ? or(humanCreatedIssueCondition(), touchedByUserCondition(companyId, filters.creationVisibilityUserId))! + : humanCreatedIssueCondition(); +} + +async function blockedInboxIssueConditions( + dbOrTx: any, + companyId: string, + filters?: IssueFilters, +) { + const conditions = [ + eq(issues.companyId, companyId), + visibleIssueCondition(), + notInArray(issues.status, [...BLOCKED_INBOX_TERMINAL_STATUSES]), + ]; + if (filters?.readCondition) conditions.push(filters.readCondition); + const creationCondition = creationVisibilityCondition(companyId, filters); + if (creationCondition) conditions.push(creationCondition); + const touchedByUserId = filters?.touchedByUserId?.trim() || undefined; + const inboxArchivedByUserId = + filters?.inboxArchivedByUserId?.trim() || undefined; + const unreadForUserId = filters?.unreadForUserId?.trim() || undefined; + const contextUserId = + unreadForUserId ?? touchedByUserId ?? inboxArchivedByUserId; + + if (filters?.createdFromIssueId) { + conditions.push(createdFromIssueCondition(companyId, filters.createdFromIssueId)); + } + if (filters?.descendantOf) { + conditions.push(sql` + ${issues.id} IN ( + WITH RECURSIVE descendants(id) AS ( + SELECT ${issues.id} + FROM ${issues} + WHERE ${issues.companyId} = ${companyId} + AND ${issues.parentId} = ${filters.descendantOf} + UNION + SELECT ${issues.id} + FROM ${issues} + JOIN descendants ON ${issues.parentId} = descendants.id + WHERE ${issues.companyId} = ${companyId} + ) + SELECT id FROM descendants + ) + `); + } + const lowTrustCondition = lowTrustBoundaryIssueCondition( + companyId, + filters?.lowTrustBoundary, + ); + if (lowTrustCondition) conditions.push(lowTrustCondition); + const statuses = parseStatusFilter(filters?.status); + if (statuses.length > 0) { + conditions.push( + statuses.length === 1 + ? eq(issues.status, statuses[0]!) + : inArray(issues.status, statuses), + ); + } + const assigneeAgentFilter = parseIssueAssigneeAgentFilter( + filters?.assigneeAgentId, + ); + assertValidAssigneeAgentFilter(assigneeAgentFilter); + if (assigneeAgentFilter === null) { + conditions.push(isNull(issues.assigneeAgentId)); + } else if (assigneeAgentFilter) { + conditions.push(eq(issues.assigneeAgentId, assigneeAgentFilter)); + } + if (filters?.participantAgentId) + conditions.push( + participatedByAgentCondition(companyId, filters.participantAgentId), + ); + if (filters?.assigneeUserId) + conditions.push(eq(issues.assigneeUserId, filters.assigneeUserId)); + if (touchedByUserId) + conditions.push(touchedByUserCondition(companyId, touchedByUserId)); + if (inboxArchivedByUserId) + conditions.push( + inboxVisibleForUserCondition(companyId, inboxArchivedByUserId), + ); + if (unreadForUserId) + conditions.push(unreadForUserCondition(companyId, unreadForUserId)); + if (filters?.projectId) + conditions.push(eq(issues.projectId, filters.projectId)); + if (filters?.workspaceId) { + conditions.push( + or( + eq(issues.executionWorkspaceId, filters.workspaceId), + eq(issues.projectWorkspaceId, filters.workspaceId), + )!, + ); + } + if (filters?.executionWorkspaceId) + conditions.push( + eq(issues.executionWorkspaceId, filters.executionWorkspaceId), + ); + if (filters?.parentId) conditions.push(eq(issues.parentId, filters.parentId)); + if (filters?.originKind) + conditions.push(eq(issues.originKind, filters.originKind)); + if (filters?.originKindPrefix) + conditions.push(like(issues.originKind, `${filters.originKindPrefix}%`)); + if (filters?.originId) conditions.push(eq(issues.originId, filters.originId)); + if (filters?.hasPlanDocument !== undefined) { + conditions.push( + hasPlanDocumentCondition(companyId, filters.hasPlanDocument), + ); + } + if (!shouldIncludePluginOperationIssues(filters)) + conditions.push(nonPluginOperationIssueCondition()); + if (filters?.labelId) { + const labeledIssueIds = await dbOrTx + .select({ issueId: issueLabels.issueId }) + .from(issueLabels) + .where( + and( + eq(issueLabels.companyId, companyId), + eq(issueLabels.labelId, filters.labelId), + ), + ); + if (labeledIssueIds.length === 0) + return { conditions: [sql`false`], contextUserId }; + conditions.push( + inArray( + issues.id, + labeledIssueIds.map((row: { issueId: string }) => row.issueId), + ), + ); + } + if ( + filters?.excludeRoutineExecutions && + !filters?.originKind && + !filters?.originId + ) { + conditions.push(ne(issues.originKind, "routine_execution")); + } + + return { conditions, contextUserId }; +} + +async function listBlockedInboxIssues( + dbOrTx: any, + companyId: string, + filters?: IssueFilters, +): Promise< + Array< + IssueWithLabelsAndRun & { + blockedBy?: IssueRelationIssueSummary[]; + blockerAttention?: IssueBlockerAttention; + reviewAttention?: IssueReviewAttention; + blockedInboxAttention: IssueBlockedInboxAttention; + liveDescendantCount?: number; + lastActivityAt: Date; + myLastTouchAt?: Date | null; + lastExternalCommentAt?: Date | null; + isUnreadForMe?: boolean; + } + > +> { + const { conditions, contextUserId } = await blockedInboxIssueConditions( + dbOrTx, + companyId, + filters, + ); + + const rows = ( + await dbOrTx + .select(issueListSelect) + .from(issues) + .where(and(...conditions)) + .orderBy( + desc(issueCanonicalLastActivityAtExpr(companyId)), + desc(issues.updatedAt), + desc(issues.id), + ) + ).map((row: any) => ({ + ...row, + description: decodeDatabaseTextPreview( + row.description, + ISSUE_LIST_DESCRIPTION_MAX_CHARS, + ), + })); + const withLabels = await withIssueLabels(dbOrTx, rows); + const withRuns = withActiveRuns( + withLabels, + await activeRunMapForIssues(dbOrTx, withLabels), + ); + if (withRuns.length === 0) return []; + + const issueIds = withRuns.map((row) => row.id); + const includeLiveDescendantSummary = + filters?.includeLiveDescendantSummary === true; + const [ + statsRows, + readRows, + lastActivityRows, + blockedByMap, + blockerAttentionByIssueId, + reviewAttentionByIssueId, + blockedInboxAttentionByIssueId, + liveDescendantCountByIssueId, + ] = await Promise.all([ + contextUserId + ? userCommentStatsForIssues(dbOrTx, companyId, contextUserId, issueIds) + : Promise.resolve([]), + contextUserId + ? userReadStatsForIssues(dbOrTx, companyId, contextUserId, issueIds) + : Promise.resolve([]), + lastActivityStatsForIssues(dbOrTx, companyId, issueIds), + blockedByMapForIssues(dbOrTx, companyId, issueIds), + listIssueBlockerAttentionMap(dbOrTx, companyId, withRuns), + listIssueReviewAttentionMap(dbOrTx, companyId, withRuns), + listIssueBlockedInboxAttentionMap(dbOrTx, companyId, withRuns), + includeLiveDescendantSummary + ? liveDescendantCountMapForIssues(dbOrTx, companyId, issueIds) + : Promise.resolve(new Map()), + ]); + + const rawSearchInput = filters?.q?.trim() ?? ""; + const rawSearch = rawSearchInput.toLowerCase(); + const commentSearchMatchIssueIds = new Set(); + if (rawSearchInput) { + const containsPattern = `%${escapeLikePattern(rawSearchInput)}%`; + for (const issueIdChunk of chunkList( + issueIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const rows = await dbOrTx + .select({ issueId: issueComments.issueId }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, companyId), + inArray(issueComments.issueId, issueIdChunk), + isNull(issueComments.deletedAt), + sql`${issueComments.body} ILIKE ${containsPattern} ESCAPE '\\'`, + ), + ); + for (const row of rows as Array<{ issueId: string }>) + commentSearchMatchIssueIds.add(row.issueId); + } + } + const statsByIssueId = new Map(statsRows.map((row) => [row.issueId, row])); + const readByIssueId = new Map( + readRows.map((row) => [row.issueId, row.myLastReadAt]), + ); + const lastActivityByIssueId = new Map( + lastActivityRows.map((row) => [row.issueId, row]), + ); + + const enriched = withRuns + .flatMap((row) => { + const blockedInboxAttention = blockedInboxAttentionByIssueId.get(row.id); + if (!blockedInboxAttention) return []; + if ( + rawSearch && + !blockedInboxSearchText(blockedInboxAttention, row).includes( + rawSearch, + ) && + !commentSearchMatchIssueIds.has(row.id) + ) + return []; + + const activity = lastActivityByIssueId.get(row.id); + const lastActivityAt = + latestIssueActivityAt( + row.updatedAt, + activity?.latestCommentAt ?? null, + activity?.latestLogAt ?? null, + ) ?? row.updatedAt; + return [ + { + ...row, + description: blockedInboxResponseDescription( + blockedInboxAttention, + row, + ), + blockedBy: blockedByMap.get(row.id) ?? [], + lastActivityAt, + ...(blockerAttentionByIssueId.has(row.id) + ? { blockerAttention: blockerAttentionByIssueId.get(row.id) } + : {}), + reviewAttention: + reviewAttentionByIssueId.get(row.id) ?? reviewAttentionNone(), + blockedInboxAttention, + ...(includeLiveDescendantSummary + ? { + liveDescendantCount: + liveDescendantCountByIssueId.get(row.id) ?? 0, + } + : {}), + ...(contextUserId + ? deriveIssueUserContext(row, contextUserId, { + isUnreadForMe: statsByIssueId.get(row.id)?.isUnreadForMe ?? false, + myLastCommentAt: + statsByIssueId.get(row.id)?.myLastCommentAt ?? null, + myLastReadAt: readByIssueId.get(row.id) ?? null, + lastExternalCommentAt: + statsByIssueId.get(row.id)?.lastExternalCommentAt ?? null, + }) + : {}), + }, + ]; + }) + .sort(compareBlockedInboxRows); + + const offset = + typeof filters?.offset === "number" && Number.isFinite(filters.offset) + ? Math.max(0, Math.floor(filters.offset)) + : 0; + const limit = + typeof filters?.limit === "number" && Number.isFinite(filters.limit) + ? Math.max(1, Math.floor(filters.limit)) + : undefined; + return limit === undefined + ? enriched.slice(offset) + : enriched.slice(offset, offset + limit); +} + +async function countBlockedInboxIssues( + dbOrTx: any, + companyId: string, + filters?: IssueFilters, +): Promise { + const { conditions } = await blockedInboxIssueConditions( + dbOrTx, + companyId, + filters, + ); + const rows = (await dbOrTx + .select() + .from(issues) + .where(and(...conditions))) as IssueRow[]; + if (rows.length === 0) return 0; + + const blockedInboxAttentionByIssueId = + await listIssueBlockedInboxAttentionMap(dbOrTx, companyId, rows); + const rawSearchInput = filters?.q?.trim() ?? ""; + const rawSearch = rawSearchInput.toLowerCase(); + const commentSearchMatchIssueIds = new Set(); + if (rawSearchInput) { + const issueIds = rows.map((row) => row.id); + const containsPattern = `%${escapeLikePattern(rawSearchInput)}%`; + for (const issueIdChunk of chunkList( + issueIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const commentRows = await dbOrTx + .select({ issueId: issueComments.issueId }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, companyId), + inArray(issueComments.issueId, issueIdChunk), + isNull(issueComments.deletedAt), + sql`${issueComments.body} ILIKE ${containsPattern} ESCAPE '\\'`, + ), + ); + for (const row of commentRows as Array<{ issueId: string }>) + commentSearchMatchIssueIds.add(row.issueId); + } + } + + return rows.reduce((count: number, row: IssueRow) => { + const attention = blockedInboxAttentionByIssueId.get(row.id); + if (!attention) return count; + if ( + rawSearch && + !blockedInboxSearchText(attention, row).includes(rawSearch) && + !commentSearchMatchIssueIds.has(row.id) + ) + return count; + return count + 1; + }, 0); +} + +export async function readIssueCommentRunLogText(run: { + runId?: string | null; + logStore: string | null; + logRef: string | null; + logBytes: number | null; +}, db?: Db) { + if (!isRunLogStoreType(run.logStore) || !run.logRef) return ""; + // A timed-out finalization leaves size unknown even when earlier entries + // exist. Read those logs within the same byte budget as a known-size log. + if (run.logBytes !== null && (!Number.isFinite(run.logBytes) || run.logBytes <= 0)) return ""; + + const logRef = run.logRef; + const storeType = run.logStore; + const store = getRunLogStore(); + let offset = 0; + let content = ""; + let nextOffset: number | undefined = 0; + const controller = new AbortController(); + let readTimer: NodeJS.Timeout | undefined; + + const readChunks = async () => { + if (db && storeType === "s3_segments") await recoverSettledNativeRunLog(db, store, logRef, undefined, controller.signal) + .catch(() => { controller.signal.throwIfAborted(); }); + while (nextOffset !== undefined) { + controller.signal.throwIfAborted(); + const remainingBytes = + ISSUE_COMMENT_RUN_LOG_DERIVATION_MAX_LOG_BYTES - + Buffer.byteLength(content, "utf8"); + if (remainingBytes <= 0) break; + const chunk = await store.read( + { store: storeType, logRef }, + { + offset, + limitBytes: Math.min(ISSUE_COMMENT_RUN_LOG_DERIVATION_CHUNK_BYTES, remainingBytes), + signal: controller.signal, + }, + ); + controller.signal.throwIfAborted(); + content += chunk.content; + nextOffset = chunk.nextOffset; + offset = chunk.nextOffset ?? 0; + } + }; + + try { + await Promise.race([ + readChunks(), + new Promise((_resolve, reject) => { + readTimer = setTimeout(() => { + const reason = new DOMException("Attribution log read timed out", "TimeoutError"); + // Cancellation closes storage work where supported, but filesystem + // I/O can delay stream destruction. Keep the response deadline too. + reject(reason); + controller.abort(reason); + }, ISSUE_COMMENT_RUN_LOG_DERIVATION_TIMEOUT_MS); + readTimer.unref?.(); + }), + ]); + } catch (err) { + // Attribution enriches already-authorized comments. Missing, failed, or + // stalled storage must not prevent listing them; keep any evidence read. + // Do not log raw provider errors, which can contain credentialed URLs. + logger.warn( + { runId: run.runId ?? undefined, logRef, status: err instanceof HttpError ? err.status : undefined }, + "could not read heartbeat run log while deriving optional issue comment metadata", + ); + } finally { + clearTimeout(readTimer); + } + + return content; +} + +/** Share task-before-comment lock order with receipt completion and task deletion. */ +async function lockIssueForCommentMutation(tx: DbTransaction, commentId: string) { + const [comment] = await tx.select({ issueId: issueComments.issueId, companyId: issueComments.companyId }) + .from(issueComments).where(eq(issueComments.id, commentId)); + if (!comment) return false; + const [issue] = await tx.select({ id: issues.id }).from(issues).where(and( + eq(issues.id, comment.issueId), eq(issues.companyId, comment.companyId), + )).for("no key update"); + return Boolean(issue); +} + +export function issueService(db: Db) { + const instanceSettings = instanceSettingsService(db); + const treeControlSvc = issueTreeControlService(db); + + function provisionalTitleFromDescription(description: string) { + const simpleTitle = description.trim().replace(/\s+/g, " ").slice(0, 120); + try { + type MarkdownNode = { + type: string; + alt?: string | null; + children?: MarkdownNode[]; + position?: { start: { offset?: number }; end: { offset?: number } }; + }; + const imageRanges: Array<{ start: number; end: number }> = []; + const imageAlts: string[] = []; + const visit = (node: MarkdownNode) => { + if (node.type === "image" || node.type === "imageReference") { + const start = node.position?.start.offset; + const end = node.position?.end.offset; + if (start !== undefined && end !== undefined) imageRanges.push({ start, end }); + if (node.alt?.trim()) imageAlts.push(node.alt.trim()); + } + node.children?.forEach(visit); + }; + visit(parseMarkdown(description) as MarkdownNode); + const withoutImages = imageRanges + .sort((a, b) => b.start - a.start) + .reduce((text, range) => `${text.slice(0, range.start)} ${text.slice(range.end)}`, description); + const plainText = markdownToPlainText(withoutImages).trim().replace(/\s+/g, " "); + const fallback = imageRanges.length > 0 + ? imageAlts.join(" ") || "Image" + : simpleTitle; + return (plainText || fallback).slice(0, 120); + } catch { + return simpleTitle; + } + } + + function normalizeCreateIssueTitle(title: string) { + return title.trim().replace(/\s+/g, " ").toLowerCase(); + } + + async function getIssueByUuid(id: string) { + const row = await retryIdempotentDatabaseOperation(() => db + .select({ ...getTableColumns(issues), externalConversationState: externalConversationStateSql() }) + .from(issues) + .where(eq(issues.id, id)) + .then((rows) => rows[0] ?? null)); + if (!row) return null; + const [enriched] = await withIssueLabels(db, [row]); + return enriched; + } + + async function getIssueByIdentifier(identifier: string) { + const row = await retryIdempotentDatabaseOperation(() => db + .select({ ...getTableColumns(issues), externalConversationState: externalConversationStateSql() }) + .from(issues) + .where(eq(issues.identifier, identifier.toUpperCase())) + .then((rows) => rows[0] ?? null)); + if (!row) return null; + const [enriched] = await withIssueLabels(db, [row]); + return enriched; + } + + async function projectHistoricalRunComments< + T extends { body: string; createdByRunId: string | null }, + >(comments: T[]): Promise { + const runIds = [ + ...new Set( + comments.flatMap((comment) => + comment.createdByRunId && comment.body === LEGACY_WITHHELD_RUN_COMMENT + ? [comment.createdByRunId] + : [], + ), + ), + ]; + if (runIds.length === 0) return comments; + const runResults = await db + .select({ id: heartbeatRuns.id, resultJson: heartbeatRuns.resultJson }) + .from(heartbeatRuns) + .where(inArray(heartbeatRuns.id, runIds)); + const resultByRunId = new Map( + runResults.map((run) => [run.id, parseObject(run.resultJson)]), + ); + return comments.map((comment) => { + if (!comment.createdByRunId) return comment; + const body = projectHistoricalHeartbeatRunComment( + comment.body, + resultByRunId.get(comment.createdByRunId), + ); + return body === comment.body ? comment : { ...comment, body }; + }); + } + + async function getCurrentScheduledRetriesForIssues( + issueIds: string[], + companyId: string, + dbOrTx: DbReader = db, + ): Promise> { + const uniqueIssueIds = [...new Set(issueIds)]; + if (uniqueIssueIds.length === 0) return new Map(); + + const contextIssueId = sql`${heartbeatRuns.contextSnapshot} ->> 'issueId'`; + const rows = await dbOrTx + .select({ + issueId: contextIssueId, + runId: heartbeatRuns.id, + status: heartbeatRuns.status, + agentId: heartbeatRuns.agentId, + agentName: agents.name, + retryOfRunId: heartbeatRuns.retryOfRunId, + scheduledRetryAt: heartbeatRuns.scheduledRetryAt, + scheduledRetryAttempt: heartbeatRuns.scheduledRetryAttempt, + scheduledRetryReason: heartbeatRuns.scheduledRetryReason, + error: heartbeatRuns.error, + errorCode: heartbeatRuns.errorCode, + }) + .from(heartbeatRuns) + .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id)) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.status, [ + "scheduled_retry", + "queued", + "running", + ]), + isNotNull(heartbeatRuns.scheduledRetryReason), + inArray(contextIssueId, uniqueIssueIds), + ), + ) + .orderBy( + sql`case ${heartbeatRuns.status} + when 'running' then 0 + when 'queued' then 1 + else 2 + end`, + asc(heartbeatRuns.scheduledRetryAt), + asc(heartbeatRuns.createdAt), + asc(heartbeatRuns.id), + ); + + const currentByIssueId = new Map(); + for (const row of rows) { + if (currentByIssueId.has(row.issueId)) continue; + const status = row.status; + if ( + status !== "scheduled_retry" && + status !== "queued" && + status !== "running" + ) + continue; + currentByIssueId.set(row.issueId, { ...row, status }); + } + return currentByIssueId; + } + + async function getCurrentScheduledRetryForIssue( + issueId: string, + companyId: string, + ): Promise { + const currentByIssueId = await getCurrentScheduledRetriesForIssues( + [issueId], + companyId, + ); + return currentByIssueId.get(issueId) ?? null; + } + + function deriveIssueCommentAuthorType(comment: { + authorType?: string | null; + authorAgentId?: string | null; + authorUserId?: string | null; + }): IssueCommentAuthorType { + const explicit = issueCommentAuthorTypeSchema.safeParse(comment.authorType); + if (explicit.success) return explicit.data; + if (comment.authorAgentId) return "agent"; + if (comment.authorUserId) return "user"; + return "system"; + } + + function assertIssueCommentAuthorTypeAllowed( + actor: { agentId?: string | null; userId?: string | null }, + authorType: IssueCommentAuthorType, + ) { + if (actor.agentId && authorType !== "agent") { + throw unprocessable("Comment authorType must match authenticated actor"); + } + if (actor.userId && authorType !== "user") { + throw unprocessable("Comment authorType must match authenticated actor"); + } + if (!actor.agentId && !actor.userId && authorType !== "system") { + throw unprocessable( + "System comments cannot use user or agent authorType without an author id", + ); + } + } + + function redactIssueComment< + T extends { + body: string; + authorType?: string | null; + authorAgentId?: string | null; + authorUserId?: string | null; + presentation?: unknown; + metadata?: unknown; + deletedAt?: Date | string | null; + deletedByType?: "agent" | "user" | null; + deletedByAgentId?: string | null; + deletedByUserId?: string | null; + deletedByRunId?: string | null; + }, + >( + comment: T, + censorUsernameInLogs: boolean, + ): T & { + authorType: IssueCommentAuthorType; + presentation: IssueCommentPresentation | null; + metadata: IssueCommentMetadata | null; + } { + const deletedAt = comment.deletedAt ?? null; + if (deletedAt) { + return { + ...comment, + authorType: deriveIssueCommentAuthorType(comment), + body: "", + presentation: null, + metadata: null, + deletedAt, + deletedByType: comment.deletedByType ?? null, + deletedByAgentId: comment.deletedByAgentId ?? null, + deletedByUserId: comment.deletedByUserId ?? null, + deletedByRunId: comment.deletedByRunId ?? null, + }; + } + + return { + ...comment, + authorType: deriveIssueCommentAuthorType(comment), + body: redactCurrentUserText(comment.body, { + enabled: censorUsernameInLogs, + }), + presentation: issueCommentPresentationSchema + .nullable() + .catch(null) + .parse(comment.presentation ?? null), + metadata: issueCommentMetadataSchema + .nullable() + .catch(null) + .parse(comment.metadata ?? null), + }; + } + + + // Persist a resolved attribution so subsequent reads stop re-scanning run + // logs (and old "Board" threads stay fixed durably). Best-effort: a write + // failure must never break the read path. The `IS NULL` guard keeps this + // idempotent and avoids clobbering a value another reader just stored. + async function persistDerivedIssueCommentAttribution( + derivedByCommentId: ReadonlyMap, + ) { + if (derivedByCommentId.size === 0) return; + // One bulk `UPDATE ... FROM (VALUES ...)` so the read path is never blocked + // on N sequential round-trips for a large legacy thread. The `IS NULL` guard + // keeps this idempotent and avoids clobbering a value another reader just + // stored. Best-effort: a write failure must never break the read path. + const rows = [...derivedByCommentId].map( + ([commentId, derived]) => + sql`(${commentId}::uuid, ${derived.derivedAuthorAgentId}::uuid, ${derived.derivedCreatedByRunId}::uuid, ${derived.derivedAuthorSource}::text)`, + ); + try { + await db.execute(sql` + UPDATE ${issueComments} AS c + SET derived_author_agent_id = v.agent_id, + derived_created_by_run_id = v.run_id, + derived_author_source = v.source + FROM (VALUES ${sql.join(rows, sql`, `)}) AS v(comment_id, agent_id, run_id, source) + WHERE c.id = v.comment_id AND c.derived_author_agent_id IS NULL + `); + } catch (err) { + logger.warn( + { err, commentIds: [...derivedByCommentId.keys()] }, + "failed to persist derived issue-comment attribution", + ); + } + } + + async function enrichCommentsWithDerivedAgentAttribution< + T extends { + id: string; + companyId: string; + issueId: string; + authorAgentId?: string | null; + authorUserId?: string | null; + createdByRunId?: string | null; + derivedAuthorAgentId?: string | null; + createdAt: Date | string; + }, + >(comments: readonly T[]) { + // Candidates: a non-human author, no stored agent, and not already resolved + // by a previous read / the backfill migration. + const preliminary = comments.filter( + (comment) => + !comment.authorAgentId && + !!comment.authorUserId && + !comment.derivedAuthorAgentId, + ); + if (preliminary.length === 0) return comments; + + const companyId = comments[0]?.companyId ?? null; + const issueId = comments[0]?.issueId ?? null; + if (!companyId || !issueId) return comments; + + // Guard: never reattribute a comment whose author maps to a genuine user + // profile. Only the non-human sentinels agents post under (e.g. + // `local-board`) are eligible — even though `local-board` is itself a row in + // the `user` table, so a plain "exists in user table" check would wrongly + // exclude every mis-attributed agent comment. + const nonSentinelAuthorUserIds = [ + ...new Set( + preliminary + .map((comment) => comment.authorUserId) + .filter( + (id): id is string => + !!id && !NON_HUMAN_SENTINEL_AUTHOR_USER_IDS.has(id), + ), + ), + ]; + const genuineUserIds = nonSentinelAuthorUserIds.length + ? new Set( + ( + await db + .select({ id: authUsers.id }) + .from(authUsers) + .where(inArray(authUsers.id, nonSentinelAuthorUserIds)) + ).map((row) => row.id), + ) + : new Set(); + // `preliminary` already guarantees a truthy `authorUserId`, so only the two + // "not a genuine user" arms are live: the explicit non-human sentinel, or an + // author id absent from the `user` table. + const candidates = preliminary.filter( + (comment) => + NON_HUMAN_SENTINEL_AUTHOR_USER_IDS.has(comment.authorUserId!) || + !genuineUserIds.has(comment.authorUserId!), + ); + if (candidates.length === 0) return comments; + + const minCommentCreatedAtMs = candidates.reduce( + (min, comment) => { + const timestamp = toTimestampMs(comment.createdAt); + if (timestamp === null) return min; + return min === null ? timestamp : Math.min(min, timestamp); + }, + null, + ); + const maxCommentCreatedAtMs = candidates.reduce( + (max, comment) => { + const timestamp = toTimestampMs(comment.createdAt); + if (timestamp === null) return max; + return max === null ? timestamp : Math.max(max, timestamp); + }, + null, + ); + if (minCommentCreatedAtMs === null || maxCommentCreatedAtMs === null) + return comments; + + const minCommentCreatedAt = new Date(minCommentCreatedAtMs).toISOString(); + const maxCommentCreatedAt = new Date( + maxCommentCreatedAtMs + ISSUE_COMMENT_RUN_LOG_DERIVATION_END_SLACK_MS, + ).toISOString(); + + // The runs the comments' own `createdByRunId` point at — fetched + // unconditionally so the lossless run-id tier resolves even when a run is + // not otherwise associated with the issue. + const ownRunIds = [ + ...new Set( + candidates + .map((comment) => comment.createdByRunId) + .filter((id): id is string => !!id), + ), + ]; + + const runs = await db + .select({ + runId: heartbeatRuns.id, + agentId: heartbeatRuns.agentId, + createdAt: heartbeatRuns.createdAt, + startedAt: heartbeatRuns.startedAt, + finishedAt: heartbeatRuns.finishedAt, + logStore: heartbeatRuns.logStore, + logRef: heartbeatRuns.logRef, + logBytes: heartbeatRuns.logBytes, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + or( + and( + or( + sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`, + sql`exists ( + select 1 + from ${activityLog} + where ${activityLog.companyId} = ${companyId} + and ${activityLog.entityType} = 'issue' + and ${activityLog.entityId} = ${issueId} + and ${activityLog.runId} = ${heartbeatRuns.id} + )`, + ), + sql`coalesce(${heartbeatRuns.finishedAt}, ${heartbeatRuns.createdAt}) >= ${minCommentCreatedAt}::timestamptz`, + sql`coalesce(${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt}) <= ${maxCommentCreatedAt}::timestamptz`, + ), + ownRunIds.length > 0 + ? inArray(heartbeatRuns.id, ownRunIds) + : sql`false`, + ), + ), + ) + .orderBy(desc(heartbeatRuns.createdAt)); + + if (runs.length === 0) return comments; + + // Pass 1: resolve the run-id tier, which never reads log bodies. Most + // comments resolve here, so we avoid object-storage reads entirely. + const runsWithoutLogs = runs.map((run) => ({ ...run, logContent: "" })); + const derivedByCommentId = new Map( + deriveIssueCommentRunLogAttribution(candidates, runsWithoutLogs), + ); + + // Pass 2: for comments still unresolved after the run-id tier, read the logs + // of any run whose window overlaps such a comment, to look for the explicit + // `comment id:` post marker. The marker is a lossless signal regardless of + // how many runs overlap, so we do not short-circuit on the single-run case. + const unresolved = candidates.filter( + (comment) => !derivedByCommentId.has(comment.id), + ); + if (unresolved.length > 0) { + const runIdsToRead = new Set(); + for (const run of runs) { + const runStartMs = toTimestampMs(run.startedAt ?? run.createdAt); + const runEndMs = toTimestampMs(run.finishedAt ?? run.createdAt); + if (runStartMs === null || runEndMs === null) continue; + for (const comment of unresolved) { + const commentCreatedAtMs = toTimestampMs(comment.createdAt); + if (commentCreatedAtMs === null) continue; + if ( + commentCreatedAtMs >= runStartMs && + commentCreatedAtMs <= + runEndMs + ISSUE_COMMENT_RUN_LOG_DERIVATION_END_SLACK_MS + ) { + runIdsToRead.add(run.runId); + break; + } + } + } + + if (runIdsToRead.size > 0) { + const runsToRead = runs.filter((run) => runIdsToRead.has(run.runId)); + const logByRunId = new Map(); + for ( + let index = 0; + index < runsToRead.length; + index += ISSUE_COMMENT_RUN_LOG_DERIVATION_MAX_PARALLEL_READS + ) { + const batch = runsToRead.slice( + index, + index + ISSUE_COMMENT_RUN_LOG_DERIVATION_MAX_PARALLEL_READS, + ); + await Promise.all( + batch.map(async (run) => { + logByRunId.set(run.runId, await readIssueCommentRunLogText(run, db)); + }), + ); + } + const runsWithLogs = runs.map((run) => ({ + ...run, + logContent: logByRunId.get(run.runId) ?? "", + })); + for (const [commentId, derived] of deriveIssueCommentRunLogAttribution( + unresolved, + runsWithLogs, + )) { + derivedByCommentId.set(commentId, derived); + } + } + } + + if (derivedByCommentId.size === 0) return comments; + + await persistDerivedIssueCommentAttribution(derivedByCommentId); + + return comments.map((comment) => { + const derived = derivedByCommentId.get(comment.id); + return derived ? { ...comment, ...derived } : comment; + }); + } + + async function isTreeHoldInteractionCheckoutAllowed( + companyId: string, + checkoutRunId: string | null, + _gate: ActiveIssueTreePauseHoldGate, + ) { + if (!checkoutRunId) return false; + const run = await db + .select({ + id: heartbeatRuns.id, + agentId: heartbeatRuns.agentId, + wakeupRequestId: heartbeatRuns.wakeupRequestId, + contextSnapshot: heartbeatRuns.contextSnapshot, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, checkoutRunId), + eq(heartbeatRuns.companyId, companyId), + ), + ) + .then((rows) => rows[0] ?? null); + const issueId = readStringFromRecord(run?.contextSnapshot, "issueId"); + if (!run || !issueId) return false; + return isVerifiedIssueTreeControlInteractionWake(db, { + companyId, + issueId, + agentId: run.agentId, + runId: run.id, + wakeupRequestId: run.wakeupRequestId, + contextSnapshot: run.contextSnapshot as + Record | null | undefined, + }); + } + + async function assertAssignableUser(companyId: string, userId: string) { + const membership = await db + .select({ id: companyMemberships.id }) + .from(companyMemberships) + .where( + and( + eq(companyMemberships.companyId, companyId), + eq(companyMemberships.principalType, "user"), + eq(companyMemberships.principalId, userId), + eq(companyMemberships.status, "active"), + ), + ) + .then((rows) => rows[0] ?? null); + if (!membership) { + throw notFound("Assignee user not found"); + } + } + + async function assertValidProjectWorkspace( + companyId: string, + projectId: string | null | undefined, + projectWorkspaceId: string, + dbOrTx: DbReader = db, + ) { + const workspace = await dbOrTx + .select({ + id: projectWorkspaces.id, + companyId: projectWorkspaces.companyId, + projectId: projectWorkspaces.projectId, + }) + .from(projectWorkspaces) + .where(eq(projectWorkspaces.id, projectWorkspaceId)) + .then((rows) => rows[0] ?? null); + if (!workspace) throw notFound("Project workspace not found"); + if (workspace.companyId !== companyId) + throw unprocessable("Project workspace must belong to same company"); + if (projectId && workspace.projectId !== projectId) { + throw unprocessable( + "Project workspace must belong to the selected project", + ); + } + return workspace; + } + + async function assertValidExecutionWorkspace( + companyId: string, + projectId: string | null | undefined, + executionWorkspaceId: string, + dbOrTx: DbReader = db, + ) { + const workspace = await dbOrTx + .select({ + id: executionWorkspaces.id, + companyId: executionWorkspaces.companyId, + projectId: executionWorkspaces.projectId, + projectWorkspaceId: executionWorkspaces.projectWorkspaceId, + }) + .from(executionWorkspaces) + .where(eq(executionWorkspaces.id, executionWorkspaceId)) + .then((rows) => rows[0] ?? null); + if (!workspace) throw notFound("Execution workspace not found"); + if (workspace.companyId !== companyId) + throw unprocessable("Execution workspace must belong to same company"); + return workspace; + } + + async function assertValidLabelIds( + companyId: string, + labelIds: string[], + dbOrTx: any = db, + ) { + if (labelIds.length === 0) return; + const existing = await dbOrTx + .select({ id: labels.id }) + .from(labels) + .where( + and(eq(labels.companyId, companyId), inArray(labels.id, labelIds)), + ); + if (existing.length !== new Set(labelIds).size) { + throw unprocessable("One or more labels are invalid for this company"); + } + } + + async function syncIssueLabels( + issueId: string, + companyId: string, + labelIds: string[], + dbOrTx: any = db, + ) { + const deduped = [...new Set(labelIds)]; + await assertValidLabelIds(companyId, deduped, dbOrTx); + await dbOrTx.delete(issueLabels).where(eq(issueLabels.issueId, issueId)); + if (deduped.length === 0) return; + await dbOrTx.insert(issueLabels).values( + deduped.map((labelId) => ({ + issueId, + labelId, + companyId, + })), + ); + } + + async function getIssueRelationSummaryMap( + companyId: string, + issueIds: string[], + dbOrTx: DbReader = db, + ): Promise> { + const uniqueIssueIds = [...new Set(issueIds)]; + const empty = new Map(); + for (const issueId of uniqueIssueIds) { + empty.set(issueId, { blockedBy: [], blocks: [] }); + } + if (uniqueIssueIds.length === 0) return empty; + + const [blockedByRows, blockingRows] = await Promise.all([ + dbOrTx + .select({ + currentIssueId: issueRelations.relatedIssueId, + relatedId: issues.id, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + priority: issues.priority, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.type, "blocks"), + inArray(issueRelations.relatedIssueId, uniqueIssueIds), + ), + ), + dbOrTx + .select({ + currentIssueId: issueRelations.issueId, + relatedId: issues.id, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + priority: issues.priority, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.relatedIssueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.type, "blocks"), + inArray(issueRelations.issueId, uniqueIssueIds), + ), + ), + ]); + + for (const row of blockedByRows) { + empty + .get(row.currentIssueId) + ?.blockedBy.push(summarizeIssueRelationRow(row)); + } + for (const row of blockingRows) { + empty + .get(row.currentIssueId) + ?.blocks.push(summarizeIssueRelationRow(row)); + } + + const terminalByRoot = await terminalExplicitBlockersByRoot( + companyId, + [...empty.values()].flatMap((relations) => relations.blockedBy), + dbOrTx, + ); + + for (const relations of empty.values()) { + relations.blockedBy.sort((a, b) => a.title.localeCompare(b.title)); + for (const blocker of relations.blockedBy) { + const terminalBlockers = terminalByRoot.get(blocker.id); + if (terminalBlockers && terminalBlockers.length > 0) { + blocker.terminalBlockers = terminalBlockers; + } + } + relations.blocks.sort((a, b) => a.title.localeCompare(b.title)); + } + + const relationSummaries: IssueRelationIssueSummary[] = []; + const collectRelationSummary = (summary: IssueRelationIssueSummary) => { + relationSummaries.push(summary); + for (const terminal of summary.terminalBlockers ?? []) + collectRelationSummary(terminal); + }; + for (const relations of empty.values()) { + for (const blocker of relations.blockedBy) + collectRelationSummary(blocker); + for (const blocking of relations.blocks) collectRelationSummary(blocking); + } + const scheduledRetryByIssueId = await getCurrentScheduledRetriesForIssues( + relationSummaries.map((summary) => summary.id), + companyId, + dbOrTx, + ); + for (const summary of relationSummaries) { + summary.scheduledRetry = scheduledRetryByIssueId.get(summary.id) ?? null; + } + + return empty; + } + + async function withIssueRelationSummaries( + companyId: string, + rows: T[], + dbOrTx: DbReader = db, + ): Promise> { + if (rows.length === 0) return []; + const relationMap = await getIssueRelationSummaryMap( + companyId, + rows.map((row) => row.id), + dbOrTx, + ); + return rows.map((row) => ({ + ...row, + ...(relationMap.get(row.id) ?? { blockedBy: [], blocks: [] }), + })); + } + + async function assertNoBlockingCycles( + companyId: string, + issueId: string, + blockerIssueIds: string[], + dbOrTx: DbReader = db, + ) { + if (blockerIssueIds.length === 0) return; + + const rows = await dbOrTx + .select({ + blockerIssueId: issueRelations.issueId, + blockedIssueId: issueRelations.relatedIssueId, + }) + .from(issueRelations) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.type, "blocks"), + ), + ); + + const adjacency = new Map(); + for (const row of rows) { + const list = adjacency.get(row.blockerIssueId) ?? []; + list.push(row.blockedIssueId); + adjacency.set(row.blockerIssueId, list); + } + + for (const blockerIssueId of blockerIssueIds) { + const queue = [...(adjacency.get(issueId) ?? [])]; + const visited = new Set([issueId]); + while (queue.length > 0) { + const current = queue.shift()!; + if (current === blockerIssueId) { + throw unprocessable("Blocking relations cannot contain cycles"); + } + if (visited.has(current)) continue; + visited.add(current); + queue.push(...(adjacency.get(current) ?? [])); + } + } + } + + async function syncBlockedByIssueIds( + issueId: string, + companyId: string, + blockedByIssueIds: string[], + actor: { agentId?: string | null; userId?: string | null } = {}, + dbOrTx: any = db, + ) { + const deduped = [...new Set(blockedByIssueIds)]; + if (deduped.some((candidate) => candidate === issueId)) { + throw unprocessable("Issue cannot be blocked by itself"); + } + + if (deduped.length > 0) { + const lockedIssueIds = [issueId, ...deduped].sort(); + await dbOrTx.execute( + sql`SELECT ${issues.id} FROM ${issues} + WHERE ${and(eq(issues.companyId, companyId), inArray(issues.id, lockedIssueIds))} + ORDER BY ${issues.id} + FOR UPDATE`, + ); + const relatedIssues = await dbOrTx + .select({ id: issues.id }) + .from(issues) + .where( + and(eq(issues.companyId, companyId), inArray(issues.id, deduped)), + ); + if (relatedIssues.length !== deduped.length) { + throw unprocessable( + "Blocked-by issues must belong to the same company", + ); + } + await assertNoBlockingCycles(companyId, issueId, deduped, dbOrTx); + } + + await dbOrTx + .delete(issueRelations) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.relatedIssueId, issueId), + eq(issueRelations.type, "blocks"), + ), + ); + + if (deduped.length === 0) return; + + await dbOrTx.insert(issueRelations).values( + deduped.map((blockerIssueId) => ({ + companyId, + issueId: blockerIssueId, + relatedIssueId: issueId, + type: "blocks", + createdByAgentId: actor.agentId ?? null, + createdByUserId: actor.userId ?? null, + })), + ); + } + + async function isTerminalOrMissingHeartbeatRun( + runId: string, + dbOrTx: DbReader = db, + ) { + return heartbeatRunIsTerminalOrMissing(dbOrTx, runId); + } + + async function adoptStaleCheckoutRun(input: { + issueId: string; + actorAgentId: string; + actorRunId: string; + expectedCheckoutRunId: string; + }) { + return db.transaction(async (tx) => { + const lockedIssue = await tx + .select({ + id: issues.id, + companyId: issues.companyId, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + }) + .from(issues) + .where(eq(issues.id, input.issueId)) + .for("update") + .then((rows) => rows[0] ?? null); + if (!lockedIssue) { + return { adopted: null, latest: null }; + } + + if ( + lockedIssue.status !== "in_progress" || + lockedIssue.assigneeAgentId !== input.actorAgentId || + lockedIssue.checkoutRunId !== input.expectedCheckoutRunId + ) { + return { adopted: null, latest: lockedIssue }; + } + + await Promise.all([ + tx.execute( + sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${input.expectedCheckoutRunId} for update`, + ), + tx.execute( + sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${input.actorRunId} for update`, + ), + ]); + const [existingRun, actorRun] = await Promise.all([ + tx + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, input.expectedCheckoutRunId)) + .then((rows) => rows[0] ?? null), + tx + .select({ status: heartbeatRuns.status }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, input.actorRunId)) + .then((rows) => rows[0] ?? null), + ]); + const stale = + !existingRun || TERMINAL_HEARTBEAT_RUN_STATUSES.has(existingRun.status); + if (isExplicitContinuationRetryClaim(lockedIssue, existingRun)) { + return { adopted: null, latest: lockedIssue }; + } + if (lockedIssue.executionRunId && lockedIssue.executionRunId !== input.expectedCheckoutRunId) { + const executionRun = await tx.select().from(heartbeatRuns) + .where(eq(heartbeatRuns.id, lockedIssue.executionRunId)).for("update") + .then(rows => rows[0] ?? null); + if (isExplicitContinuationRetryClaim(lockedIssue, executionRun)) { + return { adopted: null, latest: lockedIssue }; + } + } + const actorLive = + actorRun && !TERMINAL_HEARTBEAT_RUN_STATUSES.has(actorRun.status); + if (!stale || !actorLive) { + return { adopted: null, latest: lockedIssue }; + } + + const now = new Date(); + await notifyStatusCardIssueChange(tx, lockedIssue.companyId); + const adopted = await tx + .update(issues) + .set({ + checkoutRunId: input.actorRunId, + executionRunId: input.actorRunId, + executionLockedAt: now, + updatedAt: now, + }) + .where( + and( + eq(issues.id, input.issueId), + eq(issues.status, "in_progress"), + eq(issues.assigneeAgentId, input.actorAgentId), + eq(issues.checkoutRunId, input.expectedCheckoutRunId), + ), + ) + .returning({ + id: issues.id, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + }) + .then((rows) => rows[0] ?? null); + if (adopted) { + return { adopted, latest: adopted }; + } + + const latest = await tx + .select({ + id: issues.id, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + }) + .from(issues) + .where(eq(issues.id, input.issueId)) + .then((rows) => rows[0] ?? null); + return { adopted: null, latest }; + }); + } + + async function adoptUnownedCheckoutRun(input: { + issueId: string; + actorAgentId: string; + actorRunId: string; + }) { + return db.transaction(async (tx) => { + await tx.execute( + sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${input.actorRunId} for update`, + ); + const actorRun = await tx + .select({ status: heartbeatRuns.status, companyId: heartbeatRuns.companyId }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, input.actorRunId)) + .then((rows) => rows[0] ?? null); + if (!actorRun || TERMINAL_HEARTBEAT_RUN_STATUSES.has(actorRun.status)) + return null; + + await notifyStatusCardIssueChange(tx, actorRun.companyId); + const now = new Date(); + const adopted = await tx + .update(issues) + .set({ + checkoutRunId: input.actorRunId, + executionRunId: input.actorRunId, + executionLockedAt: now, + updatedAt: now, + }) + .where( + and( + eq(issues.id, input.issueId), + eq(issues.status, "in_progress"), + eq(issues.assigneeAgentId, input.actorAgentId), + isNull(issues.checkoutRunId), + or( + isNull(issues.executionRunId), + eq(issues.executionRunId, input.actorRunId), + ), + ), + ) + .returning({ + id: issues.id, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + }) + .then((rows) => rows[0] ?? null); + + return adopted; + }); + } + + async function clearExecutionRunIfTerminal( + issueId: string, + ): Promise { + return db.transaction(async (tx) => { + await tx.execute( + sql`select ${issues.id} from ${issues} where ${issues.id} = ${issueId} for update`, + ); + const issue = await tx + .select({ id: issues.id, companyId: issues.companyId, executionRunId: issues.executionRunId }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + if (!issue?.executionRunId) return false; + + await tx.execute( + sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${issue.executionRunId} for update`, + ); + const run = await tx + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, issue.executionRunId)) + .then((rows) => rows[0] ?? null); + if (run && !TERMINAL_HEARTBEAT_RUN_STATUSES.has(run.status)) return false; + if (isExplicitContinuationRetryClaim(issue, run)) return false; + + await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId }); + await notifyNativeStatusReassessment(tx, { companyId: issue.companyId, issueId }); + const updated = await tx + .update(issues) + .set({ + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: new Date(), + }) + .where( + and( + eq(issues.id, issueId), + eq(issues.executionRunId, issue.executionRunId), + ), + ) + .returning({ id: issues.id }) + .then((rows) => rows[0] ?? null); + + return Boolean(updated); + }); + } + + // Symmetric to clearExecutionRunIfTerminal. Clears checkoutRunId (and the + // bundled execution lock cols) when the row's checkoutRunId points at a + // heartbeat run that is terminal or no longer exists. No assignee/status + // precondition. Explicit retry claims remain owned by queue-first settlement. + async function clearCheckoutRunIfTerminal(issueId: string): Promise { + return db.transaction(async (tx) => { + await tx.execute( + sql`select ${issues.id} from ${issues} where ${issues.id} = ${issueId} for update`, + ); + const issue = await tx + .select({ + id: issues.id, + companyId: issues.companyId, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + if (!issue?.checkoutRunId) return false; + + await tx.execute( + sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${issue.checkoutRunId} for update`, + ); + const run = await tx + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, issue.checkoutRunId)) + .then((rows) => rows[0] ?? null); + if (run && !TERMINAL_HEARTBEAT_RUN_STATUSES.has(run.status)) return false; + if (isExplicitContinuationRetryClaim(issue, run)) return false; + + if ( + issue.executionRunId && + issue.executionRunId !== issue.checkoutRunId + ) { + await tx.execute( + sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${issue.executionRunId} for update`, + ); + const executionRun = await tx + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, issue.executionRunId)) + .then((rows) => rows[0] ?? null); + if ( + executionRun && + !TERMINAL_HEARTBEAT_RUN_STATUSES.has(executionRun.status) + ) + return false; + if (isExplicitContinuationRetryClaim(issue, executionRun)) return false; + } + + await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId }); + await notifyNativeStatusReassessment(tx, { companyId: issue.companyId, issueId }); + const updated = await tx + .update(issues) + .set({ + checkoutRunId: null, + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: new Date(), + }) + .where( + and( + eq(issues.id, issueId), + eq(issues.checkoutRunId, issue.checkoutRunId), + issue.executionRunId + ? eq(issues.executionRunId, issue.executionRunId) + : isNull(issues.executionRunId), + ), + ) + .returning({ id: issues.id }) + .then((rows) => rows[0] ?? null); + + return Boolean(updated); + }); + } + + async function addStopRelayCommentIfNeeded( + child: typeof issues.$inferSelect, + dbOrTx: any = db, + ) { + if ( + !child.parentId || + (child.status !== "blocked" && child.status !== "cancelled") + ) + return null; + + const relayKey = `issue-stop-relay:${child.id}:${child.status}`; + await dbOrTx.execute( + sql`select pg_advisory_xact_lock(hashtextextended(${relayKey}, 0))`, + ); + + const childIdentifier = child.identifier?.trim() || child.id; + const childPrefix = childIdentifier.split("-")[0] || "PAP"; + const body = `System relay: [${childIdentifier}](/${childPrefix}/issues/${childIdentifier}) transitioned to \`${child.status}\`.`; + const existingRelay = await dbOrTx + .select({ id: issueComments.id }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, child.companyId), + eq(issueComments.issueId, child.parentId), + eq(issueComments.authorType, "system"), + eq(issueComments.body, body), + ), + ) + .limit(1) + .then((rows: Array<{ id: string }>) => rows[0] ?? null); + if (existingRelay) return null; + + const parent = await dbOrTx + .select({ + id: issues.id, + companyId: issues.companyId, + assigneeAgentId: issues.assigneeAgentId, + status: issues.status, + }) + .from(issues) + .where( + and( + eq(issues.id, child.parentId), + eq(issues.companyId, child.companyId), + ), + ) + .then( + ( + rows: Array<{ + id: string; + companyId: string; + assigneeAgentId: string | null; + status: string; + }>, + ) => rows[0] ?? null, + ); + if (!parent) return null; + + const [comment] = await insertIssueComment(dbOrTx, { + companyId: child.companyId, + issueId: parent.id, + authorType: "system", + body, + }); + await dbOrTx + .update(issues) + .set({ updatedAt: new Date() }) + .where(eq(issues.id, parent.id)); + + return { comment, parent }; + } + + async function archiveInbox( + companyId: string, + issueId: string, + userId: string, + archivedAt: Date = new Date(), + attribution?: { + archivedByActorType: "user" | "agent"; + archivedByAgentId?: string | null; + archivedByRunId?: string | null; + }, + dbOrTx: any = db, + ) { + const runArchive = async (tx: typeof dbOrTx) => { + // Completion locks the issue before archiving. Take the FK's parent lock + // first too, or an insert can hold the archive key while waiting on that + // issue and deadlock with completion's archive UPSERT. SHARE also holds + // companyId stable; different users can still archive concurrently. + const [issue] = await tx + .select({ id: issues.id }) + .from(issues) + .where(and(eq(issues.companyId, companyId), eq(issues.id, issueId))) + .for("share"); + if (!issue) throw notFound("Issue not found"); + + const now = new Date(); + const [row] = await tx + .insert(issueInboxArchives) + .values({ + companyId, + issueId, + userId, + archivedByActorType: attribution?.archivedByActorType ?? "user", + archivedByAgentId: attribution?.archivedByAgentId ?? null, + archivedByRunId: attribution?.archivedByRunId ?? null, + archivedAt, + updatedAt: now, + }) + .onConflictDoUpdate({ + target: [ + issueInboxArchives.companyId, + issueInboxArchives.issueId, + issueInboxArchives.userId, + ], + set: { + archivedAt, + archivedByActorType: attribution?.archivedByActorType ?? "user", + archivedByAgentId: attribution?.archivedByAgentId ?? null, + archivedByRunId: attribution?.archivedByRunId ?? null, + updatedAt: now, + }, + // A request that waited behind completion must not replace the newer + // archive with its earlier request time and resurface the done task. + setWhere: lte(issueInboxArchives.archivedAt, archivedAt), + }) + .returning(); + if (row) return row; + // ON CONFLICT holds this row lock even when setWhere skips the update. + // Return the newer state, including its matching actor attribution. + const [existing] = await tx + .select() + .from(issueInboxArchives) + .where(and( + eq(issueInboxArchives.companyId, companyId), + eq(issueInboxArchives.issueId, issueId), + eq(issueInboxArchives.userId, userId), + )); + if (!existing) throw new Error("Inbox archive conflict row missing"); + return existing; + }; + return dbOrTx === db ? db.transaction(runArchive) : runArchive(dbOrTx); + } + + const service = { + clearExecutionRunIfTerminal, + clearCheckoutRunIfTerminal, + addStopRelayCommentIfNeeded, + + unreadInboxIssueIds: async (companyId: string, userId: string, readCondition: SQL) => { + // Rank the visible Mine set before counting unread, just like the inbox. + // This projection never reads descriptions, run payloads, or recovery details. + const directlyParticipated = or( + and(eq(issues.createdByUserId, userId), humanCreatedIssueCondition()), + eq(issues.assigneeUserId, userId), + ); + const participationSources = unionAll( + db.select({ id: sql`${issues.id}`.as("id") }).from(issues).where(and( + eq(issues.companyId, companyId), + directlyParticipated, + )), + db.select({ id: sql`case when length(${activityLog.entityId}) = 36 and ${activityLog.entityId} ~ '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' then ${activityLog.entityId}::uuid end`.as("id") }).from(activityLog).where(and( + userParticipationActivityCondition(companyId, userId), + notInArray(activityLog.entityId, db.select({ id: sql`${issues.id}::text` }).from(issues) + .where(and(eq(issues.companyId, companyId), directlyParticipated))), + )), + db.select({ id: sql`${issueComments.issueId}`.as("id") }).from(issueComments).where(and( + eq(issueComments.companyId, companyId), eq(issueComments.authorUserId, userId), directHumanCommentCondition(), + )), + db.select({ id: sql`${issues.id}`.as("id") }).from(issueThreadInteractions) + .innerJoin(issues, and(eq(issues.id, issueThreadInteractions.issueId), eq(issues.companyId, companyId))) + .where(pendingUserRequestRowCondition(companyId, userId)), + ).as("mine_participation_sources"); + const participated = db.selectDistinct({ id: participationSources.id }).from(participationSources).as("mine_participants"); + const eligible = db.select({ id: issues.id, updatedAt: issues.updatedAt }) + .from(participated).innerJoin(issues, eq(issues.id, sql`"mine_participants"."id"`)).where(and( + eq(issues.companyId, companyId), + visibleIssueCondition(), + isNull(issues.conversationAgentId), + nonPluginOperationIssueCondition(), + ne(issues.status, "cancelled"), + inboxVisibleForUserCondition(companyId, userId), + readCondition, + )).orderBy(asc(issues.id)).as("issues"); + const rows = await db.select({ id: eligible.id }).from(eligible) + .orderBy(desc(issueCanonicalLastActivityAtExpr(companyId)), desc(eligible.updatedAt), desc(eligible.id)).limit(100); + if (rows.length === 0) return []; + // Keep correlated predicates in WHERE (as in the list's user stats). + // A single-table Drizzle SELECT projection unqualifies nested columns. + return db.select({ id: issues.id }).from(issues).where(and( + eq(issues.companyId, companyId), + inArray(issues.id, rows.map((row) => row.id)), + unreadForUserCondition(companyId, userId), + readCondition, + )).then((unread) => unread.map((row) => row.id)); + }, + + list: async (companyId: string, filters?: IssueFilters) => { + if (filters?.sortField === "id" && filters.attention) { + throw unprocessable("ID ordering is not supported for blocked attention lists"); + } + if (filters?.afterId !== undefined && ( + !isUuidLike(filters.afterId) || filters.sortField !== "id" || + filters.sortDir !== "asc" || (filters.offset ?? 0) !== 0 + )) { + throw unprocessable("afterId requires a UUID, ascending ID order and no offset"); + } + if (filters?.attention === "blocked") { + return listBlockedInboxIssues(db, companyId, { + ...filters, + includeBlockedBy: true, + includeBlockedInboxAttention: true, + }); + } + + const conditions = [ + eq(issues.companyId, companyId), + visibleIssueCondition(), + ]; + if (filters?.issueIds) { + if (filters.issueIds.length === 0) return []; + conditions.push(inArray(issues.id, filters.issueIds)); + } + if (filters?.parentIds) { + if (filters.parentIds.length === 0) return []; + conditions.push(inArray(issues.parentId, filters.parentIds)); + } + if (!filters?.q?.trim() && !filters?.issueIds) { + conditions.push(isNull(issues.conversationAgentId)); + if (!filters?.touchedByUserId && !filters?.unreadForUserId && !filters?.inboxArchivedByUserId) { + conditions.push(nonIdleSlackIssueCondition()); + } + } + if (filters?.afterId) conditions.push(gt(issues.id, filters.afterId)); + if (filters?.readCondition) conditions.push(filters.readCondition); + const creationCondition = creationVisibilityCondition(companyId, filters); + if (creationCondition) conditions.push(creationCondition); + const assigneeAgentFilter = parseIssueAssigneeAgentFilter( + filters?.assigneeAgentId, + ); + assertValidAssigneeAgentFilter(assigneeAgentFilter); + const limit = + typeof filters?.limit === "number" && Number.isFinite(filters.limit) + ? Math.max(1, Math.floor(filters.limit)) + : undefined; + const offset = + typeof filters?.offset === "number" && Number.isFinite(filters.offset) + ? Math.max(0, Math.floor(filters.offset)) + : 0; + const touchedByUserId = filters?.touchedByUserId?.trim() || undefined; + const inboxArchivedByUserId = + filters?.inboxArchivedByUserId?.trim() || undefined; + const unreadForUserId = filters?.unreadForUserId?.trim() || undefined; + const contextUserId = + unreadForUserId ?? touchedByUserId ?? inboxArchivedByUserId; + const includeBlockedBy = filters?.includeBlockedBy === true; + const includeBlockedInboxAttention = + filters?.includeBlockedInboxAttention === true; + const includeLiveDescendantSummary = + filters?.includeLiveDescendantSummary === true; + const rawSearch = filters?.q?.trim() ?? ""; + const hasSearch = rawSearch.length > 0; + const taskSearch = parseTaskSearch(rawSearch); + if (filters?.createdFromIssueId) { + conditions.push(createdFromIssueCondition(companyId, filters.createdFromIssueId)); + } + if (filters?.descendantOf) { + conditions.push(sql` + ${issues.id} IN ( + WITH RECURSIVE descendants(id) AS ( + SELECT ${issues.id} + FROM ${issues} + WHERE ${issues.companyId} = ${companyId} + AND ${issues.parentId} = ${filters.descendantOf} + UNION + SELECT ${issues.id} + FROM ${issues} + JOIN descendants ON ${issues.parentId} = descendants.id + WHERE ${issues.companyId} = ${companyId} + ) + SELECT id FROM descendants + ) + `); + } + const lowTrustCondition = lowTrustBoundaryIssueCondition( + companyId, + filters?.lowTrustBoundary, + ); + if (lowTrustCondition) conditions.push(lowTrustCondition); + const statuses = parseStatusFilter(filters?.status); + if (statuses.length === 1) { + conditions.push(eq(issues.status, statuses[0])); + } else if (statuses.length > 1) { + conditions.push(inArray(issues.status, statuses)); + } + if (assigneeAgentFilter === null) { + conditions.push(isNull(issues.assigneeAgentId)); + } else if (assigneeAgentFilter) { + conditions.push(eq(issues.assigneeAgentId, assigneeAgentFilter)); + } + if (filters?.participantAgentId) { + conditions.push( + participatedByAgentCondition(companyId, filters.participantAgentId), + ); + } + if (filters?.assigneeUserId) { + conditions.push(eq(issues.assigneeUserId, filters.assigneeUserId)); + } + if (touchedByUserId) { + conditions.push(touchedByUserCondition(companyId, touchedByUserId)); + } + if (inboxArchivedByUserId) { + conditions.push( + inboxVisibleForUserCondition(companyId, inboxArchivedByUserId), + ); + } + if (unreadForUserId) { + conditions.push(unreadForUserCondition(companyId, unreadForUserId)); + } + if (filters?.projectId) + conditions.push(eq(issues.projectId, filters.projectId)); + if (filters?.workspaceId) { + conditions.push( + or( + eq(issues.executionWorkspaceId, filters.workspaceId), + eq(issues.projectWorkspaceId, filters.workspaceId), + )!, + ); + } + if (filters?.executionWorkspaceId) { + conditions.push( + eq(issues.executionWorkspaceId, filters.executionWorkspaceId), + ); + } + if (filters?.parentId) + conditions.push(eq(issues.parentId, filters.parentId)); + if (filters?.originKind) + conditions.push(eq(issues.originKind, filters.originKind)); + if (filters?.originKindPrefix) + conditions.push( + like(issues.originKind, `${filters.originKindPrefix}%`), + ); + if (filters?.originId) + conditions.push(eq(issues.originId, filters.originId)); + if (filters?.hasPlanDocument !== undefined) { + conditions.push( + hasPlanDocumentCondition(companyId, filters.hasPlanDocument), + ); + } + if (!shouldIncludePluginOperationIssues(filters)) { + conditions.push(nonPluginOperationIssueCondition()); + } + if (filters?.labelId) { + const labeledIssueIds = await db + .select({ issueId: issueLabels.issueId }) + .from(issueLabels) + .where( + and( + eq(issueLabels.companyId, companyId), + eq(issueLabels.labelId, filters.labelId), + ), + ); + if (labeledIssueIds.length === 0) return []; + conditions.push( + inArray( + issues.id, + labeledIssueIds.map((row) => row.issueId), + ), + ); + } + if (filters?.updatedSince) { + const since = new Date(filters.updatedSince); + if (Number.isFinite(since.getTime())) { + conditions.push(gt(issues.updatedAt, since)); + } + } + if ( + filters?.excludeRoutineExecutions && + !filters?.originKind && + !filters?.originId + ) { + conditions.push(ne(issues.originKind, "routine_execution")); + } + const priorityOrder = sql`CASE ${issues.priority} WHEN 'critical' THEN 0 WHEN 'high' THEN 1 WHEN 'medium' THEN 2 WHEN 'low' THEN 3 ELSE 4 END`; + const searchOrder = sql`-task_search.score`; + const selectedFields = filters?.listProjection ? { + ...issueListSelect, + description: sql`NULL`, + descriptionTruncated: sql`false`, + } : issueListSelect; + const issueSource = (filters?.oneChildPerParent + ? db.selectDistinctOn([issues.parentId], selectedFields) + : db.select(selectedFields)).from(issues); + const searchedSource = hasSearch + ? issueSource.innerJoin(sql`( + ${taskSearchCtes(companyId, taskSearch, true, and(...conditions), filters?.readCondition)} + SELECT m.id, ${taskSearchScore(taskSearch)} AS score FROM matched m + ) task_search`, sql`task_search.id = ${issues.id}`) + : issueSource; + const baseQuery = searchedSource + .where(and(...conditions)) + .orderBy( + ...(filters?.oneChildPerParent ? [asc(issues.parentId)] : []), + ...issueListOrderBy(companyId, { + hasSearch, + priorityOrder, + searchOrder, + sortField: filters?.sortField, + sortDir: filters?.sortDir, + }), + ); + const pageQuery = + offset > 0 + ? limit === undefined + ? baseQuery.offset(offset) + : baseQuery.limit(limit).offset(offset) + : limit === undefined + ? baseQuery + : baseQuery.limit(limit); + const rows = (await pageQuery).map((row) => ({ + ...row, + description: decodeDatabaseTextPreview( + row.description, + ISSUE_LIST_DESCRIPTION_MAX_CHARS, + ), + })); + const withLabels = await withIssueLabels(db, rows); + const runMap = filters?.listProjection === "index" ? new Map() : await activeRunMapForIssues(db, withLabels); + const withRuns = withActiveRuns(withLabels, runMap); + if (withRuns.length === 0) { + return withRuns; + } + + const issueIds = withRuns.map((row) => row.id); + const [ + statsRows, + readRows, + lastActivityRows, + archiveRows, + blockedByMap, + liveDescendantCountByIssueId, + ] = await Promise.all([ + contextUserId + ? userCommentStatsForIssues(db, companyId, contextUserId, issueIds) + : Promise.resolve([]), + contextUserId + ? userReadStatsForIssues(db, companyId, contextUserId, issueIds) + : Promise.resolve([]), + lastActivityStatsForIssues(db, companyId, issueIds), + contextUserId + ? inboxArchiveRowsForIssues(db, companyId, contextUserId, issueIds) + : Promise.resolve([]), + includeBlockedBy + ? blockedByMapForIssues(db, companyId, issueIds) + : Promise.resolve(new Map()), + includeLiveDescendantSummary + ? liveDescendantCountMapForIssues(db, companyId, issueIds) + : Promise.resolve(new Map()), + ]); + const statsByIssueId = new Map( + statsRows.map((row) => [row.issueId, row]), + ); + const lastActivityByIssueId = new Map( + lastActivityRows.map((row) => [row.issueId, row]), + ); + const archiveByIssueId = new Map( + archiveRows.map((row) => [row.issueId, row]), + ); + const [ + blockerAttentionByIssueId, + reviewAttentionByIssueId, + blockedInboxAttentionByIssueId, + ] = await Promise.all([ + filters?.listProjection === "index" ? Promise.resolve(new Map()) : listIssueBlockerAttentionMap(db, companyId, withRuns), + filters?.listProjection === "index" ? Promise.resolve(new Map()) : listIssueReviewAttentionMap(db, companyId, withRuns), + includeBlockedInboxAttention + ? listIssueBlockedInboxAttentionMap(db, companyId, withRuns) + : Promise.resolve(new Map()), + ]); + + if (!contextUserId) { + return withRuns.map((row) => { + const activity = lastActivityByIssueId.get(row.id); + const lastActivityAt = + latestIssueActivityAt( + row.updatedAt, + activity?.latestCommentAt ?? null, + activity?.latestLogAt ?? null, + ) ?? row.updatedAt; + return { + ...row, + ...(includeBlockedBy + ? { blockedBy: blockedByMap.get(row.id) ?? [] } + : {}), + lastActivityAt, + ...(blockerAttentionByIssueId.has(row.id) + ? { blockerAttention: blockerAttentionByIssueId.get(row.id) } + : {}), + reviewAttention: + reviewAttentionByIssueId.get(row.id) ?? reviewAttentionNone(), + ...(includeBlockedInboxAttention + ? { + blockedInboxAttention: + blockedInboxAttentionByIssueId.get(row.id) ?? null, + } + : {}), + ...(includeLiveDescendantSummary + ? { + liveDescendantCount: + liveDescendantCountByIssueId.get(row.id) ?? 0, + } + : {}), + }; + }); + } + + const readByIssueId = new Map( + readRows.map((row) => [row.issueId, row.myLastReadAt]), + ); + + return withRuns.map((row) => { + const activity = lastActivityByIssueId.get(row.id); + const lastActivityAt = + latestIssueActivityAt( + row.updatedAt, + activity?.latestCommentAt ?? null, + activity?.latestLogAt ?? null, + ) ?? row.updatedAt; + return { + ...row, + ...activeInboxArchiveFields( + archiveByIssueId.get(row.id), + lastActivityAt, + ), + ...(includeBlockedBy + ? { blockedBy: blockedByMap.get(row.id) ?? [] } + : {}), + lastActivityAt, + ...(blockerAttentionByIssueId.has(row.id) + ? { blockerAttention: blockerAttentionByIssueId.get(row.id) } + : {}), + reviewAttention: + reviewAttentionByIssueId.get(row.id) ?? reviewAttentionNone(), + ...(includeBlockedInboxAttention + ? { + blockedInboxAttention: + blockedInboxAttentionByIssueId.get(row.id) ?? null, + } + : {}), + ...(includeLiveDescendantSummary + ? { + liveDescendantCount: + liveDescendantCountByIssueId.get(row.id) ?? 0, + } + : {}), + ...deriveIssueUserContext(row, contextUserId, { + isUnreadForMe: statsByIssueId.get(row.id)?.isUnreadForMe ?? false, + myLastCommentAt: + statsByIssueId.get(row.id)?.myLastCommentAt ?? null, + myLastReadAt: readByIssueId.get(row.id) ?? null, + lastExternalCommentAt: + statsByIssueId.get(row.id)?.lastExternalCommentAt ?? null, + }), + }; + }); + }, + + count: async (companyId: string, filters?: IssueFilters) => { + if (filters?.attention === "blocked") { + return countBlockedInboxIssues(db, companyId, filters); + } + + const conditions = [eq(issues.companyId, companyId), visibleIssueCondition()]; + if (filters?.readCondition) conditions.push(filters.readCondition); + const creationCondition = creationVisibilityCondition(companyId, filters); + if (creationCondition) conditions.push(creationCondition); + if (!filters?.q?.trim()) { + conditions.push(isNull(issues.conversationAgentId)); + if (!filters?.touchedByUserId && !filters?.unreadForUserId && !filters?.inboxArchivedByUserId) { + conditions.push(nonIdleSlackIssueCondition()); + } + } + const statuses = parseStatusFilter(filters?.status); + if (statuses.length === 1) + conditions.push(eq(issues.status, statuses[0]!)); + else if (statuses.length > 1) + conditions.push(inArray(issues.status, statuses)); + const assigneeAgentFilter = parseIssueAssigneeAgentFilter( + filters?.assigneeAgentId, + ); + assertValidAssigneeAgentFilter(assigneeAgentFilter); + if (assigneeAgentFilter === null) { + conditions.push(isNull(issues.assigneeAgentId)); + } else if (assigneeAgentFilter) { + conditions.push(eq(issues.assigneeAgentId, assigneeAgentFilter)); + } + if (filters?.assigneeUserId) + conditions.push(eq(issues.assigneeUserId, filters.assigneeUserId)); + if (filters?.projectId) + conditions.push(eq(issues.projectId, filters.projectId)); + if (filters?.workspaceId) { + conditions.push( + or( + eq(issues.executionWorkspaceId, filters.workspaceId), + eq(issues.projectWorkspaceId, filters.workspaceId), + )!, + ); + } + if (filters?.executionWorkspaceId) + conditions.push( + eq(issues.executionWorkspaceId, filters.executionWorkspaceId), + ); + if (filters?.parentId) + conditions.push(eq(issues.parentId, filters.parentId)); + if (filters?.originKind) + conditions.push(eq(issues.originKind, filters.originKind)); + if (filters?.originKindPrefix) + conditions.push( + like(issues.originKind, `${filters.originKindPrefix}%`), + ); + if (filters?.originId) + conditions.push(eq(issues.originId, filters.originId)); + if (filters?.hasPlanDocument !== undefined) { + conditions.push( + hasPlanDocumentCondition(companyId, filters.hasPlanDocument), + ); + } + if (!shouldIncludePluginOperationIssues(filters)) + conditions.push(nonPluginOperationIssueCondition()); + const [row] = await db + .select({ count: sql`count(*)` }) + .from(issues) + .where(and(...conditions)); + return Number(row?.count ?? 0); + }, + + countUnreadTouchedByUser: async ( + companyId: string, + userId: string, + status?: string | readonly string[], + ) => { + const conditions = [ + eq(issues.companyId, companyId), + visibleIssueCondition(), + nonPluginOperationIssueCondition(), + unreadForUserCondition(companyId, userId), + ]; + const statuses = parseStatusFilter(status); + if (statuses.length === 1) { + conditions.push(eq(issues.status, statuses[0])); + } else if (statuses.length > 1) { + conditions.push(inArray(issues.status, statuses)); + } + const [row] = await db + .select({ count: sql`count(*)` }) + .from(issues) + .where(and(...conditions)); + return Number(row?.count ?? 0); + }, + + markRead: async ( + companyId: string, + issueId: string, + userId: string, + readAt: Date = new Date(), + ) => { + const now = new Date(); + const [row] = await db + .insert(issueReadStates) + .values({ + companyId, + issueId, + userId, + lastReadAt: readAt, + updatedAt: now, + }) + .onConflictDoUpdate({ + target: [ + issueReadStates.companyId, + issueReadStates.issueId, + issueReadStates.userId, + ], + set: { + lastReadAt: readAt, + updatedAt: now, + }, + }) + .returning(); + return row; + }, + + markUnread: async (companyId: string, issueId: string, userId: string) => { + const deleted = await db + .delete(issueReadStates) + .where( + and( + eq(issueReadStates.companyId, companyId), + eq(issueReadStates.issueId, issueId), + eq(issueReadStates.userId, userId), + ), + ) + .returning(); + return deleted.length > 0; + }, + + archiveInbox, + + /** + * Seed inbox archives for a batch of freshly imported issues so a company + * import does not flood the importing user's inbox. Imported issues are + * historical work, not new inbox items, but the inbox "mine" query surfaces + * every touched-and-not-archived issue; a 1000-task import would otherwise + * bury the inbox. Seeding a per-user inbox archive keeps them hidden via + * `inboxVisibleForUserCondition`, while genuine new activity on an imported + * issue still resurfaces it. This runs only on import (mirroring how + * `pauseAutomations` threads an import-only suppression) and never touches + * normal issue creation. Rows carry the same "user"-attributed shape a + * manual inbox archive writes, batched to stay under Postgres bind limits. + */ + archiveImportedInbox: async ( + companyId: string, + issueIds: string[], + userId: string, + archivedAt: Date = new Date(), + ): Promise => { + if (issueIds.length === 0) return; + const now = new Date(); + const rows = issueIds.map((issueId) => ({ + companyId, + issueId, + userId, + archivedByActorType: "user" as const, + archivedByAgentId: null, + archivedByRunId: null, + archivedAt, + createdAt: now, + updatedAt: now, + })); + await insertRowsInChunks(db, issueInboxArchives, rows); + }, + + unarchiveInbox: async ( + companyId: string, + issueId: string, + userId: string, + ) => { + const [row] = await db + .delete(issueInboxArchives) + .where( + and( + eq(issueInboxArchives.companyId, companyId), + eq(issueInboxArchives.issueId, issueId), + eq(issueInboxArchives.userId, userId), + ), + ) + .returning(); + return row ?? null; + }, + + getActiveInboxArchiveFields: async ( + issue: Pick, + userId: string, + ) => { + const [[activity], [archive]] = await Promise.all([ + lastActivityStatsForIssues(db, issue.companyId, [issue.id]), + inboxArchiveRowsForIssues(db, issue.companyId, userId, [issue.id]), + ]); + const lastActivityAt = + latestIssueActivityAt( + issue.updatedAt, + activity?.latestCommentAt ?? null, + activity?.latestLogAt ?? null, + ) ?? issue.updatedAt; + return activeInboxArchiveFields(archive, lastActivityAt); + }, + + /** + * Walk the full parent chain from `parentIssueId` (inclusive) looking for + * a still-open ancestor created by `agentId`. Used to refuse agent + * delegation cycles: an agent assigning a new child to the agent that + * created an open ancestor is handing the same work back to its own + * delegator (A→B→A hot-potato). One recursive query covers the entire + * chain regardless of depth; `UNION` (not `UNION ALL`) deduplicates + * revisited rows, so a parent graph corrupted into a loop terminates + * instead of recursing forever. + */ + findOpenAncestorCreatedByAgent: async ( + parentIssueId: string, + agentId: string, + ): Promise<{ + id: string; + identifier: string | null; + parentId: string | null; + createdByAgentId: string | null; + status: string; + } | null> => { + const rows = await db.execute(sql` + WITH RECURSIVE ancestors(id, parent_id) AS ( + SELECT id, parent_id FROM issues WHERE id = ${parentIssueId} + UNION + SELECT parent.id, parent.parent_id + FROM issues parent + JOIN ancestors ON parent.id = ancestors.parent_id + ) + SELECT i.id, i.identifier, i.parent_id, i.created_by_agent_id, i.status + FROM issues i + JOIN ancestors a ON a.id = i.id + WHERE i.created_by_agent_id = ${agentId} + AND i.status NOT IN ('done', 'cancelled') + LIMIT 1 + `); + const first = (Array.isArray(rows) ? rows[0] : null) as Record< + string, + unknown + > | null; + if (!first) return null; + return { + id: String(first.id), + identifier: + typeof first.identifier === "string" ? first.identifier : null, + parentId: typeof first.parent_id === "string" ? first.parent_id : null, + createdByAgentId: + typeof first.created_by_agent_id === "string" + ? first.created_by_agent_id + : null, + status: String(first.status), + }; + }, + + getById: async (raw: string) => { + const id = raw.trim(); + const identifier = normalizeIssueReferenceIdentifier(id); + if (identifier) { + return getIssueByIdentifier(identifier); + } + if (!isUuidLike(id)) { + return null; + } + return getIssueByUuid(id); + }, + + getByIdForUpdate: async (id: string, dbOrTx: any) => { + return dbOrTx + .select() + .from(issues) + .where(eq(issues.id, id)) + .for("update") + .then((rows: Array) => rows[0] ?? null); + }, + + getByIdentifier: async (identifier: string) => { + return getIssueByIdentifier(identifier); + }, + + getCurrentScheduledRetry: async (issueId: string) => { + const issue = await db + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + if (!issue) throw notFound("Issue not found"); + return getCurrentScheduledRetryForIssue(issue.id, issue.companyId); + }, + + getRelationSummaries: async (issueId: string) => { + const issue = await db + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + if (!issue) throw notFound("Issue not found"); + const relations = await getIssueRelationSummaryMap( + issue.companyId, + [issueId], + db, + ); + return relations.get(issueId) ?? { blockedBy: [], blocks: [] }; + }, + + getBlockerDiagnostics: async ( + issueId: string, + maxBlockers = ISSUE_BLOCKER_DIAGNOSTICS_MAX_BLOCKERS, + ) => { + const issue = await db + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + if (!issue) throw notFound("Issue not found"); + + const cappedMax = Math.max( + 0, + Math.min(maxBlockers, ISSUE_BLOCKER_DIAGNOSTICS_MAX_BLOCKERS), + ); + const blockerRows = await db + .select({ + id: issues.id, + companyId: issues.companyId, + projectId: issues.projectId, + parentId: issues.parentId, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + priority: issues.priority, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, issue.companyId), + eq(issueRelations.type, "blocks"), + eq(issueRelations.relatedIssueId, issue.id), + eq(issues.companyId, issue.companyId), + ), + ) + .orderBy(asc(issues.title), asc(issues.id)) + .limit(cappedMax + 1); + + const readiness = await listIssueDependencyReadinessMap( + db, + issue.companyId, + [issue.id], + ); + + return { + blockers: blockerRows.slice( + 0, + cappedMax, + ) as IssueBlockerDiagnosticsIssueRow[], + readiness: + readiness.get(issue.id) ?? createIssueDependencyReadiness(issue.id), + truncated: blockerRows.length > cappedMax, + }; + }, + + getWakeDiagnostics: async ( + issueId: string, + opts?: { + maxWakeRequests?: number; + maxActivityRecords?: number; + lookbackDays?: number; + }, + ) => { + const issue = await db + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + if (!issue) throw notFound("Issue not found"); + + const maxWakeRequests = Math.max( + 0, + Math.min( + opts?.maxWakeRequests ?? ISSUE_WAKE_DIAGNOSTICS_MAX_WAKE_REQUESTS, + ISSUE_WAKE_DIAGNOSTICS_MAX_WAKE_REQUESTS, + ), + ); + const maxActivityRecords = Math.max( + 0, + Math.min( + opts?.maxActivityRecords ?? + ISSUE_WAKE_DIAGNOSTICS_MAX_ACTIVITY_RECORDS, + ISSUE_WAKE_DIAGNOSTICS_MAX_ACTIVITY_RECORDS, + ), + ); + const lookbackDays = Math.max( + 1, + Math.min( + opts?.lookbackDays ?? ISSUE_WAKE_DIAGNOSTICS_LOOKBACK_DAYS, + ISSUE_WAKE_DIAGNOSTICS_LOOKBACK_DAYS, + ), + ); + const since = new Date(Date.now() - lookbackDays * 24 * 60 * 60 * 1000); + + const wakeRows = await db + .select({ + agentId: agentWakeupRequests.agentId, + source: agentWakeupRequests.source, + reason: agentWakeupRequests.reason, + status: agentWakeupRequests.status, + coalescedCount: agentWakeupRequests.coalescedCount, + runId: agentWakeupRequests.runId, + requestedAt: agentWakeupRequests.requestedAt, + claimedAt: agentWakeupRequests.claimedAt, + finishedAt: agentWakeupRequests.finishedAt, + error: agentWakeupRequests.error, + }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, issue.companyId), + gte(agentWakeupRequests.requestedAt, since), + wakeRequestTargetsIssue(issue.id), + ), + ) + .orderBy( + desc(agentWakeupRequests.requestedAt), + desc(agentWakeupRequests.createdAt), + ) + .limit(maxWakeRequests + 1); + + const activityRows = await db + .select({ + action: activityLog.action, + entityType: activityLog.entityType, + entityId: activityLog.entityId, + agentId: activityLog.agentId, + runId: activityLog.runId, + details: activityLog.details, + createdAt: activityLog.createdAt, + }) + .from(activityLog) + .where( + and( + eq(activityLog.companyId, issue.companyId), + gte(activityLog.createdAt, since), + inArray(activityLog.action, [ + ...ISSUE_WAKE_DIAGNOSTICS_ACTIVITY_ACTIONS, + ]), + wakeDiagnosticActivityTargetsIssue(issue.id), + ), + ) + .orderBy(desc(activityLog.createdAt)) + .limit(maxActivityRecords + 1); + + return { + wakeRequests: wakeRows.slice( + 0, + maxWakeRequests, + ) as IssueWakeDiagnosticsWakeRequestRow[], + activityRecords: activityRows.slice( + 0, + maxActivityRecords, + ) as IssueWakeDiagnosticsActivityRow[], + truncatedWakeRequests: wakeRows.length > maxWakeRequests, + truncatedActivityRecords: activityRows.length > maxActivityRecords, + caps: { + maxWakeRequests: ISSUE_WAKE_DIAGNOSTICS_MAX_WAKE_REQUESTS, + maxActivityRecords: ISSUE_WAKE_DIAGNOSTICS_MAX_ACTIVITY_RECORDS, + lookbackDays: ISSUE_WAKE_DIAGNOSTICS_LOOKBACK_DAYS, + }, + }; + }, + + getSubtreeDiagnostics: async ( + issueId: string, + opts?: { + maxDepth?: number; + maxNodes?: number; + maxBlockersPerNode?: number; + maxWakeRequestsPerNode?: number; + maxActivityRecordsPerNode?: number; + lookbackDays?: number; + }, + ) => { + const issue = await db + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + if (!issue) throw notFound("Issue not found"); + + const maxDepth = Math.max( + 0, + Math.min( + opts?.maxDepth ?? ISSUE_SUBTREE_DIAGNOSTICS_MAX_DEPTH, + ISSUE_SUBTREE_DIAGNOSTICS_MAX_DEPTH, + ), + ); + const maxNodes = Math.max( + 1, + Math.min( + opts?.maxNodes ?? ISSUE_SUBTREE_DIAGNOSTICS_MAX_NODES, + ISSUE_SUBTREE_DIAGNOSTICS_MAX_NODES, + ), + ); + const maxBlockersPerNode = Math.max( + 0, + Math.min( + opts?.maxBlockersPerNode ?? + ISSUE_SUBTREE_DIAGNOSTICS_MAX_BLOCKERS_PER_NODE, + ISSUE_SUBTREE_DIAGNOSTICS_MAX_BLOCKERS_PER_NODE, + ), + ); + const maxWakeRequestsPerNode = Math.max( + 0, + Math.min( + opts?.maxWakeRequestsPerNode ?? + ISSUE_SUBTREE_DIAGNOSTICS_MAX_WAKE_REQUESTS_PER_NODE, + ISSUE_SUBTREE_DIAGNOSTICS_MAX_WAKE_REQUESTS_PER_NODE, + ), + ); + const maxActivityRecordsPerNode = Math.max( + 0, + Math.min( + opts?.maxActivityRecordsPerNode ?? + ISSUE_SUBTREE_DIAGNOSTICS_MAX_ACTIVITY_RECORDS_PER_NODE, + ISSUE_SUBTREE_DIAGNOSTICS_MAX_ACTIVITY_RECORDS_PER_NODE, + ), + ); + const lookbackDays = Math.max( + 1, + Math.min( + opts?.lookbackDays ?? ISSUE_WAKE_DIAGNOSTICS_LOOKBACK_DAYS, + ISSUE_WAKE_DIAGNOSTICS_LOOKBACK_DAYS, + ), + ); + const since = new Date(Date.now() - lookbackDays * 24 * 60 * 60 * 1000); + const sinceIso = since.toISOString(); + + const rawSubtreeRows = + await db.execute(sql` + WITH RECURSIVE issue_tree AS ( + SELECT + id, + company_id, + project_id, + parent_id, + identifier, + title, + status, + priority, + assignee_agent_id, + assignee_user_id, + created_at, + updated_at, + 0 AS depth, + ARRAY[id] AS path + FROM issues + WHERE company_id = ${issue.companyId} + AND id = ${issue.id} + AND hidden_at IS NULL + AND harness_kind IS NULL + UNION ALL + SELECT + child.id, + child.company_id, + child.project_id, + child.parent_id, + child.identifier, + child.title, + child.status, + child.priority, + child.assignee_agent_id, + child.assignee_user_id, + child.created_at, + child.updated_at, + issue_tree.depth + 1, + issue_tree.path || child.id + FROM issues child + JOIN issue_tree ON child.parent_id = issue_tree.id + WHERE child.company_id = ${issue.companyId} + AND child.hidden_at IS NULL + AND child.harness_kind IS NULL + AND issue_tree.depth < ${maxDepth + 1} + AND NOT child.id = ANY(issue_tree.path) + ) + SELECT + id, + company_id AS "companyId", + project_id AS "projectId", + parent_id AS "parentId", + identifier, + title, + status, + priority, + assignee_agent_id AS "assigneeAgentId", + assignee_user_id AS "assigneeUserId", + created_at AS "createdAt", + updated_at AS "updatedAt", + depth::int AS depth + FROM issue_tree + ORDER BY depth ASC, created_at ASC, id ASC + LIMIT ${maxNodes + 1} + `); + const subtreeRows = Array.from(rawSubtreeRows).map((row) => ({ + ...row, + depth: Number(row.depth), + })); + const rowsWithinDepth = subtreeRows.filter( + (row) => row.depth <= maxDepth, + ); + const nodes = rowsWithinDepth.slice( + 0, + maxNodes, + ) as IssueSubtreeDiagnosticsIssueRow[]; + const truncatedNodes = rowsWithinDepth.length > maxNodes; + const truncatedDepth = + truncatedNodes || subtreeRows.some((row) => row.depth > maxDepth); + const nodeIds = nodes.map((node) => node.id); + + const readiness = + nodeIds.length > 0 + ? await listIssueDependencyReadinessMap(db, issue.companyId, nodeIds) + : new Map(); + const blockersByIssueId = new Map< + string, + IssueSubtreeDiagnosticsBlockerRow[] + >(); + const wakeRequestsByIssueId = new Map< + string, + IssueSubtreeDiagnosticsWakeRequestRow[] + >(); + const activityRecordsByIssueId = new Map< + string, + IssueSubtreeDiagnosticsActivityRow[] + >(); + const truncatedBlockerIssueIds = new Set(); + const truncatedWakeIssueIds = new Set(); + const truncatedActivityIssueIds = new Set(); + + if (nodeIds.length > 0) { + const nodeIdValues = sql.join( + nodeIds.map((id) => sql`${id}`), + sql`, `, + ); + const rawBlockerRows = Array.from( + await db.execute(sql` + WITH blocker_rows AS ( + SELECT + blocker.id, + blocker.company_id AS "companyId", + blocker.project_id AS "projectId", + blocker.parent_id AS "parentId", + blocker.identifier, + blocker.title, + blocker.status, + blocker.priority, + blocker.assignee_agent_id AS "assigneeAgentId", + blocker.assignee_user_id AS "assigneeUserId", + relation.related_issue_id AS "blockedIssueId", + relation.created_at AS "relationCreatedAt", + row_number() OVER ( + PARTITION BY relation.related_issue_id + ORDER BY blocker.title ASC, blocker.id ASC + )::int AS "rowNumber" + FROM issue_relations relation + INNER JOIN issues blocker ON blocker.id = relation.issue_id + WHERE relation.company_id = ${issue.companyId} + AND relation.type = 'blocks' + AND blocker.company_id = ${issue.companyId} + AND blocker.hidden_at IS NULL + AND blocker.harness_kind IS NULL + AND relation.related_issue_id::text IN (${nodeIdValues}) + ) + SELECT * + FROM blocker_rows + WHERE "rowNumber" <= ${maxBlockersPerNode + 1} + ORDER BY "blockedIssueId" ASC, "rowNumber" ASC + `), + ) as IssueSubtreeDiagnosticsBlockerResultRow[]; + for (const row of rawBlockerRows) { + const normalized = { ...row, rowNumber: Number(row.rowNumber) }; + if (normalized.rowNumber > maxBlockersPerNode) { + truncatedBlockerIssueIds.add(normalized.blockedIssueId); + continue; + } + const rows = blockersByIssueId.get(normalized.blockedIssueId) ?? []; + rows.push(normalized); + blockersByIssueId.set(normalized.blockedIssueId, rows); + } + + const wakeTargetIssueIdSql = sql` + coalesce( + wake.payload ->> 'issueId', + wake.payload ->> 'taskId', + wake.payload -> '_paperclipWakeContext' ->> 'issueId', + wake.payload -> '_paperclipWakeContext' ->> 'taskId' + ) + `; + const rawWakeRows = Array.from( + await db.execute(sql` + WITH wake_rows AS ( + SELECT + ${wakeTargetIssueIdSql} AS "issueId", + wake.agent_id AS "agentId", + wake.source, + wake.reason, + wake.status, + wake.coalesced_count AS "coalescedCount", + wake.run_id AS "runId", + wake.requested_at AS "requestedAt", + wake.claimed_at AS "claimedAt", + wake.finished_at AS "finishedAt", + wake.error, + row_number() OVER ( + PARTITION BY ${wakeTargetIssueIdSql} + ORDER BY wake.requested_at DESC, wake.created_at DESC + )::int AS "rowNumber" + FROM agent_wakeup_requests wake + WHERE wake.company_id = ${issue.companyId} + AND wake.requested_at >= ${sinceIso}::timestamptz + AND ${wakeTargetIssueIdSql} IN (${nodeIdValues}) + ) + SELECT * + FROM wake_rows + WHERE "rowNumber" <= ${maxWakeRequestsPerNode + 1} + ORDER BY "issueId" ASC, "requestedAt" DESC + `), + ) as IssueSubtreeDiagnosticsWakeRequestResultRow[]; + for (const row of rawWakeRows) { + const normalized = { ...row, rowNumber: Number(row.rowNumber) }; + if (normalized.rowNumber > maxWakeRequestsPerNode) { + truncatedWakeIssueIds.add(normalized.issueId); + continue; + } + const rows = wakeRequestsByIssueId.get(normalized.issueId) ?? []; + rows.push(normalized); + wakeRequestsByIssueId.set(normalized.issueId, rows); + } + + const activityTargetIssueIdSql = sql` + coalesce( + CASE WHEN activity.entity_type = 'issue' THEN activity.entity_id ELSE NULL END, + activity.details ->> 'issueId', + activity.details ->> 'rootIssueId' + ) + `; + const activityActionValues = sql.join( + ISSUE_WAKE_DIAGNOSTICS_ACTIVITY_ACTIONS.map( + (action) => sql`${action}`, + ), + sql`, `, + ); + const rawActivityRows = Array.from( + await db.execute(sql` + WITH activity_rows AS ( + SELECT + ${activityTargetIssueIdSql} AS "issueId", + activity.action, + activity.entity_type AS "entityType", + activity.entity_id AS "entityId", + activity.agent_id AS "agentId", + activity.run_id AS "runId", + activity.details, + activity.created_at AS "createdAt", + row_number() OVER ( + PARTITION BY ${activityTargetIssueIdSql} + ORDER BY activity.created_at DESC, activity.id DESC + )::int AS "rowNumber" + FROM activity_log activity + WHERE activity.company_id = ${issue.companyId} + AND activity.created_at >= ${sinceIso}::timestamptz + AND activity.action IN (${activityActionValues}) + AND ${activityTargetIssueIdSql} IN (${nodeIdValues}) + ) + SELECT * + FROM activity_rows + WHERE "rowNumber" <= ${maxActivityRecordsPerNode + 1} + ORDER BY "issueId" ASC, "createdAt" DESC + `), + ) as IssueSubtreeDiagnosticsActivityResultRow[]; + for (const row of rawActivityRows) { + const normalized = { ...row, rowNumber: Number(row.rowNumber) }; + if (normalized.rowNumber > maxActivityRecordsPerNode) { + truncatedActivityIssueIds.add(normalized.issueId); + continue; + } + const rows = activityRecordsByIssueId.get(normalized.issueId) ?? []; + rows.push(normalized); + activityRecordsByIssueId.set(normalized.issueId, rows); + } + } + + return { + nodes, + blockersByIssueId, + readinessByIssueId: readiness, + wakeRequestsByIssueId, + activityRecordsByIssueId, + truncatedNodes, + truncatedDepth, + truncatedBlockerIssueIds, + truncatedWakeIssueIds, + truncatedActivityIssueIds, + caps: { + maxDepth, + maxNodes, + maxBlockersPerNode, + maxWakeRequestsPerNode, + maxActivityRecordsPerNode, + lookbackDays, + }, + }; + }, + + + + getDependencyReadiness: async ( + issueId: string, + dbOrTx: any = db, + options: { lockSources?: boolean } = {}, + ) => { + const issue = await dbOrTx + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, issueId)) + .then( + (rows: Array<{ id: string; companyId: string }>) => rows[0] ?? null, + ); + if (!issue) throw notFound("Issue not found"); + const readiness = await listIssueDependencyReadinessMap( + dbOrTx, + issue.companyId, + [issueId], + options.lockSources, + ); + return readiness.get(issueId) ?? createIssueDependencyReadiness(issueId); + }, + + listDependencyReadiness: async ( + companyId: string, + issueIds: string[], + dbOrTx: any = db, + ) => { + return listIssueDependencyReadinessMap(dbOrTx, companyId, issueIds); + }, + + listBlockerAttention: async ( + companyId: string, + issueRows: IssueBlockerAttentionInputNode[], + dbOrTx: any = db, + ) => { + return listIssueBlockerAttentionMap(dbOrTx, companyId, issueRows); + }, + + listReviewAttention: async ( + companyId: string, + issueRows: IssueReviewAttentionInput[], + dbOrTx: any = db, + ) => { + return listIssueReviewAttentionMap(dbOrTx, companyId, issueRows); + }, + + listWakeableBlockedDependents: async (blockerIssueId: string) => { + const blockerIssue = await db + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, blockerIssueId)) + .then((rows) => rows[0] ?? null); + if (!blockerIssue) return []; + + const candidates = await db + .select({ + id: issues.id, + assigneeAgentId: issues.assigneeAgentId, + status: issues.status, + blockedTransitionAt: issues.blockedTransitionAt, + }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.relatedIssueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, blockerIssue.companyId), + eq(issueRelations.type, "blocks"), + eq(issueRelations.issueId, blockerIssueId), + isNull(issues.conversationAgentId), + ), + ); + if (candidates.length === 0) return []; + + const wakeableCandidates = candidates.filter( + (candidate) => + candidate.assigneeAgentId && + !["backlog", "done", "cancelled"].includes(candidate.status), + ); + if (wakeableCandidates.length === 0) return []; + + // Defer to the unified readiness check so that a dependent only fires when + // (a) every blocker is done AND (b) every done blocker's workspace has + // recorded a successful workspace_finalize. The finalize hook also calls + // this function on completion, so a wake initially gated by an in-flight + // sync-back will re-fire once the restore lands locally. + const readinessMap = await listIssueDependencyReadinessMap( + db, + blockerIssue.companyId, + wakeableCandidates.map((candidate) => candidate.id), + ); + + return wakeableCandidates + .map((candidate) => { + const readiness = + readinessMap.get(candidate.id) ?? + createIssueDependencyReadiness(candidate.id); + return { candidate, readiness }; + }) + .filter( + ({ readiness }) => + readiness.isDependencyReady && readiness.blockerIssueIds.length > 0, + ) + .map(({ candidate, readiness }) => ({ + id: candidate.id, + assigneeAgentId: candidate.assigneeAgentId!, + blockerIssueIds: readiness.blockerIssueIds, + blockedTransitionAt: candidate.blockedTransitionAt, + })); + }, + + getWakeableParentAfterChildCompletion: async ( + parentIssueId: string, + completedChildResult?: { issueId: string; summary: string | null } | null, + ) => { + const parent = await db + .select({ + id: issues.id, + conversationAgentId: issues.conversationAgentId, + originKind: issues.originKind, + assigneeAgentId: issues.assigneeAgentId, + status: issues.status, + companyId: issues.companyId, + }) + .from(issues) + .where(eq(issues.id, parentIssueId)) + .then((rows) => rows[0] ?? null); + if (!parent || parent.conversationAgentId || !parent.assigneeAgentId || ["backlog", "cancelled"].includes(parent.status) || + (parent.status === "done" && parent.originKind !== "onboarding_first_task")) { + return null; + } + + const children = await db + .select({ + id: issues.id, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + priority: issues.priority, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + updatedAt: issues.updatedAt, + }) + .from(issues) + .where( + and( + eq(issues.companyId, parent.companyId), + eq(issues.parentId, parentIssueId), + ), + ) + .orderBy(asc(issues.issueNumber), asc(issues.createdAt)); + if (children.length === 0) return null; + if ( + !children.every( + (child) => child.status === "done" || child.status === "cancelled", + ) + ) { + return null; + } + + const childIdsForSummaries = children + .slice(0, MAX_CHILD_COMPLETION_SUMMARIES) + .map((child) => child.id); + const commentRows = + childIdsForSummaries.length > 0 + ? await db + .select({ + issueId: issueComments.issueId, + body: issueComments.body, + createdAt: issueComments.createdAt, + }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, parent.companyId), + inArray(issueComments.issueId, childIdsForSummaries), + isNull(issueComments.deletedAt), + ), + ) + .orderBy(desc(issueComments.createdAt), desc(issueComments.id)) + : []; + const latestCommentByIssueId = new Map(); + for (const comment of commentRows) { + if (!latestCommentByIssueId.has(comment.issueId)) { + latestCommentByIssueId.set(comment.issueId, comment.body); + } + } + const childIssueSummaries: ChildIssueCompletionSummary[] = children + .slice(0, MAX_CHILD_COMPLETION_SUMMARIES) + .map((child) => ({ + ...child, + summary: truncateInlineSummary( + child.id === completedChildResult?.issueId + ? (completedChildResult.summary ?? + latestCommentByIssueId.get(child.id)) + : latestCommentByIssueId.get(child.id), + ), + })); + + return { + onboardingCompletion: parent.originKind === "onboarding_first_task", + id: parent.id, + assigneeAgentId: parent.assigneeAgentId, + childIssueIds: children.map((child) => child.id), + childIssueSummaries, + childIssueSummaryTruncated: + children.length > childIssueSummaries.length, + }; + }, + + createChild: async (parentIssueId: string, data: IssueChildCreateInput) => { + const parent = await db + .select() + .from(issues) + .where(eq(issues.id, parentIssueId)) + .then((rows) => rows[0] ?? null); + if (!parent) throw notFound("Parent issue not found"); + await assertExecutionTaskParent(db, parent.companyId, parent.id); + + const idempotencyKey = data.idempotencyKey?.trim(); + if (idempotencyKey) { + const existingChild = await db + .select({ issue: issues }) + .from(issueCreateIdempotencyKeys) + .innerJoin(issues, eq(issueCreateIdempotencyKeys.issueId, issues.id)) + .where( + and( + eq(issueCreateIdempotencyKeys.companyId, parent.companyId), + eq(issueCreateIdempotencyKeys.idempotencyKey, idempotencyKey), + ), + ) + .limit(1) + .then((rows) => rows[0]?.issue ?? null); + if (existingChild) { + if (existingChild.parentId !== parent.id) { + throw conflict( + "Child creation idempotency key belongs to another parent issue", + ); + } + await data.assertCanReuseIssue?.(existingChild); + assertQualityReviewPolicyReuse(data.executionPolicy, existingChild.executionPolicy); + data.onDeduplicated?.("idempotency_key"); + const [enriched] = await withIssueLabels(db, [existingChild]); + const [withRelations] = await withIssueRelationSummaries( + parent.companyId, + [enriched], + db, + ); + return { + issue: withRelations, + parentBlockerAdded: false, + }; + } + } + + const [{ childCount }] = await db + .select({ childCount: sql`count(*)::int` }) + .from(issues) + .where( + and( + eq(issues.companyId, parent.companyId), + eq(issues.parentId, parent.id), + notInArray(issues.status, ["done", "cancelled"]), + ), + ); + if (childCount >= MAX_CHILD_ISSUES_CREATED_BY_HELPER) { + throw unprocessable( + `Parent issue already has the maximum ${MAX_CHILD_ISSUES_CREATED_BY_HELPER} active child issues for this helper`, + ); + } + + const { + acceptanceCriteria, + blockParentUntilDone, + executionWorkspaceInheritanceMode = "linkage", + actorAgentId, + actorUserId, + ...issueData + } = data; + + const inheritStrategyOnly = + executionWorkspaceInheritanceMode === "strategy_only"; + // A child may target another project. Parent workspace identity is only + // valid inside the parent's project, so do not forward it across that + // boundary; create() then resolves the target project's own workspaces. + const childProjectId = issueData.projectId ?? parent.projectId; + const childInheritsParentProject = childProjectId === parent.projectId; + const hasExplicitExecutionWorkspaceOverride = + issueData.projectWorkspaceId !== undefined || + issueData.executionWorkspaceId !== undefined || + issueData.executionWorkspacePreference !== undefined || + issueData.executionWorkspaceSettings !== undefined; + const inheritedPreRealizationWorkspaceSettings = + inheritStrategyOnly && !hasExplicitExecutionWorkspaceOverride + ? buildPreRealizationExecutionWorkspaceSettings( + parent.executionWorkspaceSettings, + ) + : null; + let child = await issueService(db).create(parent.companyId, { + ...issueData, + parentId: parent.id, + projectId: childProjectId, + projectWorkspaceId: + issueData.projectWorkspaceId !== undefined + ? issueData.projectWorkspaceId + : (inheritStrategyOnly && childInheritsParentProject + ? parent.projectWorkspaceId + : undefined), + goalId: issueData.goalId ?? parent.goalId, + actorResponsibleUserId: issueData.actorResponsibleUserId ?? null, + trustExplicitResponsibleUserId: + issueData.trustExplicitResponsibleUserId === true, + requestDepth: clampIssueRequestDepth( + Math.max( + clampIssueRequestDepth(parent.requestDepth) + 1, + issueData.requestDepth ?? 0, + ), + ), + description: appendAcceptanceCriteriaToDescription( + issueData.description, + acceptanceCriteria, + ), + ...(inheritedPreRealizationWorkspaceSettings + ? { + executionWorkspaceSettings: + inheritedPreRealizationWorkspaceSettings, + } + : {}), + ...(inheritStrategyOnly + ? { skipExecutionWorkspaceInheritance: true } + : { inheritExecutionWorkspaceFromIssueId: parent.id }), + }); + + if (blockParentUntilDone) { + const existingBlockers = await db + .select({ blockerIssueId: issueRelations.issueId }) + .from(issueRelations) + .where( + and( + eq(issueRelations.companyId, parent.companyId), + eq(issueRelations.relatedIssueId, parent.id), + eq(issueRelations.type, "blocks"), + ), + ); + await syncBlockedByIssueIds( + parent.id, + parent.companyId, + [ + ...new Set([ + ...existingBlockers.map((row) => row.blockerIssueId), + child.id, + ]), + ], + { agentId: actorAgentId ?? null, userId: actorUserId ?? null }, + ); + [child] = await withIssueRelationSummaries( + parent.companyId, + [child], + db, + ); + } + + return { + issue: child, + parentBlockerAdded: Boolean(blockParentUntilDone), + }; + }, + + decomposeAcceptedPlan: async ( + sourceIssueId: string, + data: AcceptedPlanDecompositionInput, + ) => { + const sourceIssue = await db + .select({ + id: issues.id, + companyId: issues.companyId, + projectId: issues.projectId, + goalId: issues.goalId, + }) + .from(issues) + .where(eq(issues.id, sourceIssueId)) + .then((rows) => rows[0] ?? null); + if (!sourceIssue) throw notFound("Source issue not found"); + + const requestFingerprint = + createAcceptedPlanDecompositionRequestFingerprint({ + acceptedPlanRevisionId: data.acceptedPlanRevisionId, + children: data.children, + }); + + const initialClaim = await db.transaction(async (tx) => { + await tx.execute( + sql`select ${issues.id} from ${issues} where ${issues.id} = ${sourceIssue.id} for update`, + ); + + const belongsToPlanDocument = await tx + .select({ revisionId: documentRevisions.id }) + .from(issueDocuments) + .innerJoin( + documentRevisions, + eq(issueDocuments.documentId, documentRevisions.documentId), + ) + .where( + and( + eq(issueDocuments.companyId, sourceIssue.companyId), + eq(issueDocuments.issueId, sourceIssue.id), + eq(issueDocuments.key, "plan"), + eq(documentRevisions.id, data.acceptedPlanRevisionId), + ), + ) + .then((rows) => rows[0] ?? null); + if (!belongsToPlanDocument) { + throw unprocessable( + "acceptedPlanRevisionId must belong to the source issue's plan document", + ); + } + + const acceptedInteraction = await findAcceptedPlanDocumentInteraction( + tx, + { + companyId: sourceIssue.companyId, + sourceIssueId: sourceIssue.id, + acceptedPlanRevisionId: data.acceptedPlanRevisionId, + }, + ); + if (!acceptedInteraction) { + throw unprocessable( + "acceptedPlanRevisionId must have an accepted plan confirmation", + ); + } + + const existing = await tx + .select() + .from(issuePlanDecompositions) + .where( + and( + eq(issuePlanDecompositions.companyId, sourceIssue.companyId), + eq(issuePlanDecompositions.sourceIssueId, sourceIssue.id), + eq( + issuePlanDecompositions.acceptedPlanRevisionId, + data.acceptedPlanRevisionId, + ), + ), + ) + .then((rows) => rows[0] ?? null); + + const now = new Date(); + if (!existing) { + const [created] = await tx + .insert(issuePlanDecompositions) + .values({ + companyId: sourceIssue.companyId, + sourceIssueId: sourceIssue.id, + acceptedPlanRevisionId: data.acceptedPlanRevisionId, + acceptedInteractionId: acceptedInteraction.id, + status: "in_flight", + requestFingerprint, + requestedChildCount: data.children.length, + requestedChildren: data.children as unknown as Record< + string, + unknown + >[], + childIssueIds: [], + ownerAgentId: data.actorAgentId ?? null, + ownerUserId: data.actorUserId ?? null, + ownerRunId: data.actorRunId ?? null, + updatedAt: now, + }) + .returning(); + if (!created) + throw new Error( + "Failed to create accepted-plan decomposition claim", + ); + return created; + } + + if (existing.requestFingerprint !== requestFingerprint) { + throw conflict( + "Accepted-plan decomposition already exists for this revision with a different child set", + ); + } + + return existing; + }); + + let currentClaim = initialClaim; + const newlyCreatedIssues: Array = []; + + while (true) { + const step = await db.transaction(async (tx) => { + await tx.execute( + sql`select ${issuePlanDecompositions.id} + from ${issuePlanDecompositions} + where ${issuePlanDecompositions.id} = ${currentClaim.id} + for update`, + ); + + const claim = await tx + .select() + .from(issuePlanDecompositions) + .where(eq(issuePlanDecompositions.id, currentClaim.id)) + .then((rows) => rows[0] ?? null); + if (!claim) + throw notFound("Accepted-plan decomposition claim not found"); + if (claim.requestFingerprint !== requestFingerprint) { + throw conflict( + "Accepted-plan decomposition already exists for this revision with a different child set", + ); + } + + const existingChildIssueIds = normalizeIssuePlanDecompositionChildIds( + claim.childIssueIds, + ); + if ( + claim.status === "completed" || + existingChildIssueIds.length >= data.children.length + ) { + const nextIds = existingChildIssueIds.slice( + 0, + data.children.length, + ); + if ( + claim.status === "completed" && + nextIds.length === data.children.length + ) { + return { + claim, + createdIssue: null, + }; + } + + const completedAt = claim.completedAt ?? new Date(); + const ownerPatch = await resolveAcceptedPlanClaimOwner({ + dbOrTx: tx, + claim, + actorAgentId: data.actorAgentId, + actorUserId: data.actorUserId, + actorRunId: data.actorRunId, + }); + const [completed] = await tx + .update(issuePlanDecompositions) + .set({ + status: "completed", + childIssueIds: nextIds, + completedAt, + ...ownerPatch, + updatedAt: completedAt, + }) + .where(eq(issuePlanDecompositions.id, claim.id)) + .returning(); + if (!completed) + throw new Error( + "Failed to complete accepted-plan decomposition claim", + ); + return { + claim: completed, + createdIssue: null, + }; + } + + const nextChildInput = data.children[existingChildIssueIds.length]; + if (!nextChildInput) { + throw new Error( + "Accepted-plan decomposition child cursor moved past the requested children", + ); + } + + const createdChild = await issueService( + tx as unknown as Db, + ).createChild(sourceIssue.id, { + ...nextChildInput, + executionWorkspaceInheritanceMode: "strategy_only", + }); + const nextIds = [...existingChildIssueIds, createdChild.issue.id]; + const now = new Date(); + const nextStatus = + nextIds.length === data.children.length ? "completed" : "in_flight"; + const ownerPatch = await resolveAcceptedPlanClaimOwner({ + dbOrTx: tx, + claim, + actorAgentId: data.actorAgentId, + actorUserId: data.actorUserId, + actorRunId: data.actorRunId, + }); + const [updatedClaim] = await tx + .update(issuePlanDecompositions) + .set({ + status: nextStatus, + childIssueIds: nextIds, + completedAt: nextStatus === "completed" ? now : null, + ...ownerPatch, + updatedAt: now, + }) + .where(eq(issuePlanDecompositions.id, claim.id)) + .returning(); + if (!updatedClaim) + throw new Error( + "Failed to persist accepted-plan decomposition progress", + ); + return { + claim: updatedClaim, + createdIssue: createdChild.issue, + }; + }); + + currentClaim = step.claim; + if (step.createdIssue) { + newlyCreatedIssues.push(step.createdIssue); + } + if (step.claim.status === "completed") break; + } + + const childIssueIds = normalizeIssuePlanDecompositionChildIds( + currentClaim.childIssueIds, + ); + const childIssueRows = + childIssueIds.length > 0 + ? await db + .select() + .from(issues) + .where( + and( + eq(issues.companyId, sourceIssue.companyId), + inArray(issues.id, childIssueIds), + ), + ) + : []; + const childIssueMap = new Map(childIssueRows.map((row) => [row.id, row])); + const orderedChildIssues = childIssueIds + .map((childIssueId) => childIssueMap.get(childIssueId)) + .filter((row): row is typeof issues.$inferSelect => Boolean(row)); + + const decomposition = serializeAcceptedPlanDecomposition(currentClaim); + + return { + decomposition, + childIssueIds: decomposition.childIssueIds, + childIssues: orderedChildIssues, + newlyCreatedIssues, + }; + }, + + listAcceptedPlanDecompositions: async (sourceIssueId: string) => { + const sourceIssue = await db + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, sourceIssueId)) + .then((rows) => rows[0] ?? null); + if (!sourceIssue) return []; + + const rows = await db + .select({ + decomposition: issuePlanDecompositions, + revisionNumber: documentRevisions.revisionNumber, + }) + .from(issuePlanDecompositions) + .leftJoin( + documentRevisions, + eq( + documentRevisions.id, + issuePlanDecompositions.acceptedPlanRevisionId, + ), + ) + .where( + and( + eq(issuePlanDecompositions.companyId, sourceIssue.companyId), + eq(issuePlanDecompositions.sourceIssueId, sourceIssue.id), + ), + ) + .orderBy(desc(issuePlanDecompositions.createdAt)); + + if (rows.length === 0) return []; + + const allChildIds = new Set(); + for (const row of rows) { + for (const childId of normalizeIssuePlanDecompositionChildIds( + row.decomposition.childIssueIds, + )) { + allChildIds.add(childId); + } + } + + const childIssueRows = + allChildIds.size > 0 + ? await db + .select({ + id: issues.id, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + priority: issues.priority, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + }) + .from(issues) + .where( + and( + eq(issues.companyId, sourceIssue.companyId), + inArray(issues.id, Array.from(allChildIds)), + ), + ) + : []; + const childIssueMap = new Map(childIssueRows.map((row) => [row.id, row])); + + return rows.map((row) => { + const decomposition = serializeAcceptedPlanDecomposition( + row.decomposition, + ); + const childIds = decomposition.childIssueIds; + return { + ...decomposition, + acceptedPlanRevisionNumber: row.revisionNumber ?? null, + childIssues: childIds + .map((childId) => childIssueMap.get(childId) ?? null) + .filter( + (entry): entry is NonNullable => entry !== null, + ), + }; + }); + }, + + listConversations: async (companyId: string, userId: string) => db.select().from(issues).where(and( + eq(issues.companyId, companyId), eq(issues.conversationUserId, userId), isNotNull(issues.conversationAgentId), + )).orderBy(desc(issues.updatedAt), asc(issues.id)), + + getConversation: async (companyId: string, agentId: string, userId: string) => db.select().from(issues).where(and( + eq(issues.companyId, companyId), eq(issues.conversationAgentId, agentId), eq(issues.conversationUserId, userId), + )).then((rows) => rows[0] ?? null), + + create: async ( + companyId: string, + data: IssueCreateInput, + dbOrTx: Db | DbTransaction = db, + ) => { + const { + workspaceSelection, workspaceSelectionSource = "explicit", workspaceSelectionActor, + initialPlan, + conversationHandoff, + labelIds: inputLabelIds, + blockedByIssueIds, + inheritExecutionWorkspaceFromIssueId, + skipExecutionWorkspaceInheritance, + watchdog, + watchdogActorRunId, + actorRunId, + actorResponsibleUserId, + trustExplicitResponsibleUserId, + idempotencyKey: rawIdempotencyKey, + allowDuplicate, + assertCanReuseIssue, + onDeduplicated, + ...issueData + } = data; + const explicitTitle = issueData.title?.trim(); + const provisionalTitle = issueData.description + ? provisionalTitleFromDescription(issueData.description) + : undefined; + const resolvedTitle = explicitTitle || provisionalTitle; + if (!resolvedTitle) throw unprocessable("Provide a title or task description"); + const titleNeedsGeneration = !explicitTitle; + // A prompt prefix is not a task identity: distinct requests can share it. + const deduplicateByTitle = allowDuplicate === false && !titleNeedsGeneration; + issueData.title = resolvedTitle; + const isolatedWorkspacesEnabled = ( + await instanceSettings.getExperimental() + ).enableIsolatedWorkspaces; + if (!isolatedWorkspacesEnabled) { + delete issueData.executionWorkspaceId; + delete issueData.executionWorkspacePreference; + delete issueData.executionWorkspaceSettings; + } + if (data.assigneeAgentId && data.assigneeUserId) { + throw unprocessable("Issue can only have one assignee"); + } + if (data.assigneeAgentId) { + await assertAssignableAgent(db, companyId, data.assigneeAgentId, { + kind: "work", + }); + } + if (data.assigneeUserId) { + await assertAssignableUser(companyId, data.assigneeUserId); + } + if ( + data.status === "in_progress" && + !data.assigneeAgentId && + !data.assigneeUserId + ) { + throw unprocessable("in_progress issues require an assignee"); + } + const persist = async (tx: DbTransaction) => { + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`issue-privacy-tree:${companyId}`}, 0))`); + await assertExecutionTaskParent(tx as unknown as Db, companyId, issueData.parentId); + if (issueData.conversationAgentId && issueData.conversationUserId) { + const identity = `conversation:${companyId}:${issueData.conversationAgentId}:${issueData.conversationUserId}`; + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${identity}, 0))`); + const [existing] = await tx.select().from(issues).where(and(eq(issues.companyId, companyId), + eq(issues.conversationAgentId, issueData.conversationAgentId), eq(issues.conversationUserId, issueData.conversationUserId))); + if (existing) { + const [enriched] = await withIssueLabels(tx, [existing]); + const [withRelations] = await withIssueRelationSummaries(companyId, [enriched], tx); + return withRelations; + } + } + const idempotencyKey = rawIdempotencyKey?.trim() || null; + const normalizedTitle = normalizeCreateIssueTitle(resolvedTitle); + if (deduplicateByTitle) { + const titleGuardKey = `issue-create:title:${companyId}:${issueData.parentId ?? "root"}:${normalizedTitle}`; + await tx.execute( + sql`select pg_advisory_xact_lock(hashtextextended(${titleGuardKey}, 0))`, + ); + } + if (idempotencyKey) { + const idempotencyGuardKey = `issue-create:idempotency:${companyId}:${idempotencyKey}`; + await tx.execute( + sql`select pg_advisory_xact_lock(hashtextextended(${idempotencyGuardKey}, 0))`, + ); + } + + let existingIssue: typeof issues.$inferSelect | undefined; + let deduplicationReason: + "idempotency_key" | "recent_open_title" | null = null; + if (idempotencyKey) { + const idempotencyKeyRetentionCutoff = new Date( + Date.now() - ISSUE_CREATE_IDEMPOTENCY_KEY_RETENTION_MS, + ); + await tx.execute(sql` + delete from ${issueCreateIdempotencyKeys} + where ${issueCreateIdempotencyKeys.id} in ( + select ${issueCreateIdempotencyKeys.id} + from ${issueCreateIdempotencyKeys} + where ${issueCreateIdempotencyKeys.companyId} = ${companyId} + and ${issueCreateIdempotencyKeys.createdAt} < ${idempotencyKeyRetentionCutoff.toISOString()}::timestamptz + order by ${issueCreateIdempotencyKeys.createdAt} asc, ${issueCreateIdempotencyKeys.id} asc + limit ${ISSUE_CREATE_IDEMPOTENCY_KEY_CLEANUP_BATCH_SIZE} + ) + `); + + [existingIssue] = await tx + .select() + .from(issueCreateIdempotencyKeys) + .innerJoin( + issues, + eq(issueCreateIdempotencyKeys.issueId, issues.id), + ) + .where( + and( + eq(issueCreateIdempotencyKeys.companyId, companyId), + eq(issueCreateIdempotencyKeys.idempotencyKey, idempotencyKey), + ), + ) + .limit(1) + .then((rows) => rows.map((row) => row.issues)); + if (existingIssue) deduplicationReason = "idempotency_key"; + } + if (!existingIssue && deduplicateByTitle) { + [existingIssue] = await tx + .select() + .from(issues) + .where( + and( + eq(issues.companyId, companyId), + issueData.parentId + ? eq(issues.parentId, issueData.parentId) + : isNull(issues.parentId), + isNull(issues.hiddenAt), + notInArray(issues.status, ["done", "cancelled"]), + gte( + issues.createdAt, + new Date(Date.now() - 48 * 60 * 60 * 1000), + ), + sql`lower(regexp_replace(btrim(${issues.title}), '\\s+', ' ', 'g')) = ${normalizedTitle}`, + ), + ) + .orderBy(asc(issues.createdAt), asc(issues.id)) + .limit(1); + if (existingIssue) deduplicationReason = "recent_open_title"; + } + if (existingIssue) { + // A duplicate may have a different scope or assignee than the proposed task. + await assertCanReuseIssue?.(existingIssue); + assertQualityReviewPolicyReuse(issueData.executionPolicy, existingIssue.executionPolicy); + if (idempotencyKey) { + await tx + .insert(issueCreateIdempotencyKeys) + .values({ companyId, idempotencyKey, issueId: existingIssue.id }) + .onConflictDoNothing(); + } + if (deduplicationReason) onDeduplicated?.(deduplicationReason); + const [enriched] = await withIssueLabels(tx, [existingIssue]); + const [withRelations] = await withIssueRelationSummaries( + companyId, + [enriched], + tx, + ); + return withRelations; + } + + if (issueData.status === "done" && requiredQualityReview(issueData.executionPolicy)) { + throw unprocessable("A new task with required quality review cannot start completed", { code: "required_quality_review_incomplete" }); + } + await assertOnboardingFirstTaskCreationAllowed(tx as unknown as Db, { + companyId, + actorAgentId: issueData.createdByAgentId, + actorRunId, + blockedByIssueIds, + }); + + const defaultCompanyGoal = await getDefaultCompanyGoal(tx, companyId); + let projectWorkspaceId = issueData.projectWorkspaceId ?? null; + let executionWorkspaceId = issueData.executionWorkspaceId ?? null; + let executionWorkspacePreference = + issueData.executionWorkspacePreference ?? null; + let executionWorkspaceSettings = + (issueData.executionWorkspaceSettings as + Record | null | undefined) ?? null; + let inheritedWorkspaceSelection = false; + const workspaceInheritanceIssueId = skipExecutionWorkspaceInheritance + ? null + : (inheritExecutionWorkspaceFromIssueId ?? + issueData.parentId ?? + null); + const hasExplicitExecutionWorkspaceOverride = + (workspaceSelection !== undefined && workspaceSelectionSource === "explicit") || + issueData.projectWorkspaceId !== undefined || + issueData.executionWorkspaceId !== undefined || + issueData.executionWorkspacePreference !== undefined || + issueData.executionWorkspaceSettings !== undefined; + if (workspaceInheritanceIssueId) { + const workspaceSource = await getWorkspaceInheritanceIssue( + tx, + companyId, + workspaceInheritanceIssueId, + ); + if (issueData.projectId == null && workspaceSource.projectId) { + issueData.projectId = workspaceSource.projectId; + } + // Workspace linkage is only inheritable inside the source project. A + // cross-project child (for example, a Paperclip ID issue created from + // a Paperclip App parent) must fall through to its own project's + // default workspaces, otherwise the inherited ids fail the + // project-match assertions below and the create is impossible without + // the caller naming the target workspaces explicitly. + const inheritsSourceProject = + issueData.projectId == null || + issueData.projectId === workspaceSource.projectId; + if ( + inheritsSourceProject && + issueData.projectWorkspaceId === undefined && + projectWorkspaceId == null && + workspaceSource.projectWorkspaceId + ) { + projectWorkspaceId = workspaceSource.projectWorkspaceId; + inheritedWorkspaceSelection = true; + } + if ( + inheritsSourceProject && + !hasExplicitExecutionWorkspaceOverride && + workspaceSource.executionWorkspaceId + ) { + const sourceWorkspace = await tx + .select({ + id: executionWorkspaces.id, + mode: executionWorkspaces.mode, + }) + .from(executionWorkspaces) + .where( + eq( + executionWorkspaces.id, + workspaceSource.executionWorkspaceId, + ), + ) + .then((rows) => rows[0] ?? null); + if (sourceWorkspace) { + executionWorkspaceId = sourceWorkspace.id; + inheritedWorkspaceSelection = true; + executionWorkspacePreference = "reuse_existing"; + executionWorkspaceSettings = { + ...((workspaceSource.executionWorkspaceSettings as + Record | null | undefined) ?? {}), + mode: issueExecutionWorkspaceModeForPersistedWorkspace( + sourceWorkspace.mode, + ), + }; + } + } + } + const workspaceActor: AuthorizationActor = workspaceSelectionActor ?? (issueData.createdByUserId + ? { type: "board", userId: issueData.createdByUserId, companyIds: [companyId] } + : { type: "agent", agentId: issueData.createdByAgentId ?? issueData.assigneeAgentId, companyId }); + let workspaceIntent: TaskWorkspaceIntent | null = null; + if (workspaceSelection && (workspaceSelectionSource === "explicit" || !inheritedWorkspaceSelection)) { + const projection = await executionWorkspaceService(db).validateSelection({ companyId, + actor: workspaceActor, + selection: workspaceSelection, parentIssueId: issueData.parentId, + taskCreation: { parentIssueId: issueData.parentId, projectId: issueData.projectId, + assigneeAgentId: issueData.assigneeAgentId, executionPolicy: issueData.executionPolicy }, + assigneeAgentId: issueData.assigneeAgentId, assigneeUserId: issueData.assigneeUserId, assigneeAdapterOverrides: issueData.assigneeAdapterOverrides, projectId: issueData.projectId, executionPolicy: issueData.executionPolicy }, tx); + projectWorkspaceId = projection.projectWorkspaceId; + executionWorkspaceId = projection.executionWorkspaceId; + executionWorkspacePreference = projection.executionWorkspacePreference; + executionWorkspaceSettings = projection.executionWorkspaceSettings; + workspaceIntent = { version: 1, selection: workspaceSelection, source: workspaceSelectionSource }; + } + if (executionWorkspaceId) { + const workspace = await assertValidExecutionWorkspace(companyId, null, executionWorkspaceId, tx); + projectWorkspaceId = workspace.projectWorkspaceId; + } + const projectGoalId = await getProjectDefaultGoalId( + tx, + companyId, + issueData.projectId, + ); + // Cache the project policy lookup for this insert so the default + // workspace-settings block does not re-query the project row. + let projectPolicyCached: ReturnType< + typeof parseProjectExecutionWorkspacePolicy + > | null = null; + let projectPolicyLoaded = false; + const loadProjectPolicyOnce = async () => { + if (projectPolicyLoaded) return projectPolicyCached; + projectPolicyLoaded = true; + if (!issueData.projectId) return null; + const projectRow = await tx + .select({ + executionWorkspacePolicy: projects.executionWorkspacePolicy, + }) + .from(projects) + .where( + and( + eq(projects.id, issueData.projectId), + eq(projects.companyId, companyId), + ), + ) + .then((rows) => rows[0] ?? null); + projectPolicyCached = parseProjectExecutionWorkspacePolicy( + projectRow?.executionWorkspacePolicy, + ); + return projectPolicyCached; + }; + + if ( + executionWorkspaceSettings == null && + executionWorkspaceId == null && + issueData.projectId + ) { + executionWorkspaceSettings = + defaultIssueExecutionWorkspaceSettingsForProject( + gateProjectExecutionWorkspacePolicy( + await loadProjectPolicyOnce(), + isolatedWorkspacesEnabled, + ), + ) as Record | null; + } + if (!executionWorkspaceId && !projectWorkspaceId && issueData.projectId && workspaceIntent?.selection.kind !== "task_directory") { + const project = await tx + .select({ + executionWorkspacePolicy: projects.executionWorkspacePolicy, + }) + .from(projects) + .where( + and( + eq(projects.id, issueData.projectId), + eq(projects.companyId, companyId), + ), + ) + .then((rows) => rows[0] ?? null); + const projectPolicy = parseProjectExecutionWorkspacePolicy( + project?.executionWorkspacePolicy, + ); + projectWorkspaceId = await resolveProjectDefaultWorkspaceId(tx, { + companyId, projectId: issueData.projectId, policy: projectPolicy, + }); + if (projectWorkspaceId && isolatedWorkspacesEnabled) { + const sourceStrategy = await executionWorkspaceService(db).resolveSourceWorkspaceStrategy({ companyId, + assigneeAgentId: issueData.assigneeAgentId, assigneeAdapterOverrides: issueData.assigneeAdapterOverrides, + executionWorkspaceSettings, executionWorkspacePolicy: projectPolicy, sourceWorkspaceId: projectWorkspaceId, + typedSelection: false, projectId: issueData.projectId, executionPolicy: issueData.executionPolicy }, tx); + if (sourceStrategy === "git_worktree") workspaceIntent = { version: 1, source: "project", + selection: { kind: "configured_source", projectWorkspaceId, mode: "managed_isolated" } }; + } + } + if (projectWorkspaceId && workspaceSelectionActor) { + const [readableSource] = await tx.select({ id: projectWorkspaces.id, projectId: projectWorkspaces.projectId, executionWorkspacePolicy: projects.executionWorkspacePolicy }).from(projectWorkspaces) + .innerJoin(projects, eq(projects.id, projectWorkspaces.projectId)) + .where(and(eq(projectWorkspaces.id, projectWorkspaceId), eq(projectWorkspaces.companyId, companyId), await projectReadSqlCondition(tx, workspaceActor))); + if (!readableSource) throw notFound("Workspace source is unavailable or inaccessible"); + await executionWorkspaceService(db).resolveSourceWorkspaceStrategy({ companyId, + assigneeAgentId: issueData.assigneeAgentId, assigneeAdapterOverrides: issueData.assigneeAdapterOverrides, + executionWorkspaceSettings, executionWorkspacePolicy: readableSource.executionWorkspacePolicy, sourceWorkspaceId: projectWorkspaceId, + typedSelection: Boolean(workspaceIntent), projectId: issueData.projectId, executionPolicy: issueData.executionPolicy }, tx); + await assertTaskWorkspaceSourceProjectAccess(tx, workspaceActor, companyId, readableSource.projectId); + } + if (executionWorkspaceId && workspaceSelectionActor) { + const [readableWorkspace] = await tx.select({ id: executionWorkspaces.id, projectId: executionWorkspaces.projectId }).from(executionWorkspaces) + .where(and(eq(executionWorkspaces.id, executionWorkspaceId), eq(executionWorkspaces.companyId, companyId), await executionWorkspaceReadSqlCondition(tx, workspaceActor))); + if (!readableWorkspace) throw notFound("Workspace is unavailable or inaccessible"); + await assertExecutionWorkspaceRecordAccess(tx, workspaceActor, companyId, readableWorkspace.id); + } + if (projectWorkspaceId) { + await assertValidProjectWorkspace( + companyId, + null, + projectWorkspaceId, + tx, + ); + } + if ( + isolatedWorkspacesEnabled && + issueData.executionWorkspaceSettings !== undefined + ) { + assertExplicitPinnedWorktreeIssueRunnable({ + projectId: issueData.projectId ?? null, + projectWorkspaceId, + executionWorkspaceId, + executionWorkspacePreference, + executionWorkspaceSettings: issueData.executionWorkspaceSettings, + }); + } + // Self-correcting counter: use MAX(issue_number) + 1 if the counter + // has drifted below the actual max, preventing identifier collisions. + const [maxRow] = await tx + .select({ + maxNum: sql`coalesce(max(${issues.issueNumber}), 0)`, + }) + .from(issues) + .where(eq(issues.companyId, companyId)); + const currentMax = maxRow?.maxNum ?? 0; + + const [company] = await tx + .update(companies) + .set({ + issueCounter: sql`greatest(${companies.issueCounter}, ${currentMax}) + 1`, + }) + .where(eq(companies.id, companyId)) + .returning({ + issueCounter: companies.issueCounter, + issuePrefix: companies.issuePrefix, + }); + + const issueNumber = company.issueCounter; + const identifier = `${company.issuePrefix}-${issueNumber}`; + const responsibleUserId = await resolveResponsibleUserIdForIssueCreate( + tx, + companyId, + { + explicitResponsibleUserId: issueData.responsibleUserId ?? null, + createdByUserId: issueData.createdByUserId ?? null, + parentId: issueData.parentId ?? null, + originKind: issueData.originKind ?? "manual", + originRunId: issueData.originRunId ?? null, + actorRunId: actorRunId ?? null, + actorResponsibleUserId: actorResponsibleUserId ?? null, + trustExplicitResponsibleUserId: + trustExplicitResponsibleUserId === true, + }, + ); + + let visibility = issueData.visibility ?? "open"; + let privacyRootIssueId: string | null = null; + // Provenance survives standalone chat handoffs. This is an access edge, + // independent of the product's structural parent/child relationship. + const handoffSource = conversationHandoff ? await lockAcceptedChatHandoffSource(tx, { + companyId, ...conversationHandoff, + userId: issueData.createdByUserId, agentId: issueData.createdByAgentId, + }) : null; + let privacyParentIssueId = issueData.parentId ?? handoffSource?.id ?? null; + const sourceRunId = issueData.originRunId ?? actorRunId; + if (!privacyParentIssueId && sourceRunId) { + const [sourceRun] = await tx.select().from(heartbeatRuns) + .where(and(eq(heartbeatRuns.id, sourceRunId), eq(heartbeatRuns.companyId, companyId))); + const sourceId = sourceRun?.nativeIssueId ?? sourceRun?.issueId ?? sourceRun?.contextSnapshot?.issueId; + if (typeof sourceId === "string") privacyParentIssueId = sourceId; + } + let inheritedPrivateSubtree = false; + if (privacyParentIssueId) { + const [source] = await tx.select().from(issues) + .where(and(eq(issues.id, privacyParentIssueId), eq(issues.companyId, companyId))); + if (!source) privacyParentIssueId = null; + else if (source.visibility === "private" || (source.projectId && await tx.select({ id: projects.id }).from(projects) + .where(and(eq(projects.id, source.projectId), eq(projects.visibility, "private"))).limit(1).then(rows => rows.length > 0))) { + visibility = "private"; + privacyRootIssueId = source.privacyRootIssueId ?? source.id; + inheritedPrivateSubtree = true; + } + } + if (visibility === "private" && !privacyRootIssueId) { + issueData.id ??= randomUUID(); + privacyRootIssueId = issueData.id; + } + + if (issueData.projectId) { + const selectedProject = await tx + .select({ visibility: projects.visibility }) + .from(projects) + .where(and(eq(projects.id, issueData.projectId), eq(projects.companyId, companyId))) + .then((rows) => rows[0] ?? null); + if (!selectedProject) throw notFound("Project not found"); + if (selectedProject.visibility === "private") { + visibility = "private"; + if (!privacyRootIssueId) { + issueData.id ??= randomUUID(); + privacyRootIssueId = issueData.id; + } + } + } + + if (visibility === "private" && !issueData.projectId && !inheritedPrivateSubtree) { + if (!responsibleUserId) { + throw unprocessable("Private tasks require a responsible user"); + } + const personalProject = await ensurePersonalPrivateProject(tx, companyId, responsibleUserId); + issueData.projectId = personalProject.id; + } + + const values = { + ...issueData, + workspaceSelection: workspaceIntent, + visibility, + privacyRootIssueId, + privacyParentIssueId, + titleNeedsGeneration, + originRunId: issueData.originRunId ?? actorRunId ?? null, + responsibleUserId, + requestDepth: clampIssueRequestDepth(issueData.requestDepth), + originKind: issueData.originKind ?? "manual", + goalId: resolveIssueGoalId({ + projectId: issueData.projectId, + goalId: issueData.goalId, + projectGoalId, + defaultGoalId: defaultCompanyGoal?.id ?? null, + }), + projectWorkspaceId, + executionWorkspaceId, + executionWorkspacePreference, + executionWorkspaceSettings, + companyId, + issueNumber, + identifier, + } as typeof issues.$inferInsert; + if (values.status === "in_progress" && !values.startedAt) { + values.startedAt = new Date(); + } + if (values.status === "done") { + values.completedAt = new Date(); + } + if (values.status === "cancelled") { + values.cancelledAt = new Date(); + } + Object.assign( + values, + buildInitialIssueMonitorFields({ + policy: normalizeIssueExecutionPolicy( + issueData.executionPolicy ?? null, + ), + status: values.status ?? "backlog", + assigneeAgentId: values.assigneeAgentId ?? null, + assigneeUserId: values.assigneeUserId ?? null, + }), + ); + + if (values.parentId || values.executionWorkspaceId || values.status === "done" || values.status === "cancelled") { + await notifyTerminalWorkspaceWork(tx, values.executionWorkspaceId ? `workspace:${values.executionWorkspaceId}` + : values.parentId ? `issue:${values.parentId}` : undefined); + } + if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx, { companyId: values.companyId }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); + const [issue] = await tx.insert(issues).values(values).returning(); + await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); + if (issue.visibility === "private" && !inheritedPrivateSubtree && issueData.createdByAgentId) { + await tx.insert(issueAccessGrants).values({ + issueId: issue.id, + subjectType: "agent", + subjectId: issueData.createdByAgentId, + source: "explicit", + grantedByAgentId: issueData.createdByAgentId, + }).onConflictDoNothing(); + } + await ensureAssignmentIssueAccessGrant(tx, issue, null, { + agentId: issueData.createdByAgentId ?? null, + userId: issueData.createdByUserId ?? null, + }); + await recordChatHandoff(tx, issue, actorRunId); + if (handoffSource) await recordAcceptedChatHandoff(tx, issue, handoffSource); + if (idempotencyKey) { + await tx.insert(issueCreateIdempotencyKeys).values({ + companyId, + idempotencyKey, + issueId: issue.id, + }); + } + if (watchdog) { + await upsertIssueWatchdogForIssue(tx, companyId, issue.id, { + agentId: watchdog.agentId, + instructions: watchdog.instructions, + actor: { + agentId: issueData.createdByAgentId ?? null, + userId: issueData.createdByUserId ?? null, + runId: watchdogActorRunId ?? null, + }, + }); + } + if (inputLabelIds) { + await syncIssueLabels(issue.id, companyId, inputLabelIds, tx); + } + if (blockedByIssueIds !== undefined) { + await syncBlockedByIssueIds( + issue.id, + companyId, + blockedByIssueIds, + { + agentId: issueData.createdByAgentId ?? null, + userId: issueData.createdByUserId ?? null, + }, + tx, + ); + } + if (initialPlan?.trim()) { + await documentService(tx as unknown as Db).upsertIssueDocument({ + issueId: issue.id, key: "plan", title: "Plan", format: "markdown", body: initialPlan, + createdByAgentId: issueData.createdByAgentId, createdByUserId: issueData.createdByUserId, + createdByRunId: actorRunId, + sourceTrust: issue.sourceTrust, + }); + } + const [enriched] = await withIssueLabels(tx, [issue]); + const [withRelations] = await withIssueRelationSummaries( + companyId, + [enriched], + tx, + ); + return withRelations; + }; + if (dbOrTx === db) return db.transaction(persist); + return persist(dbOrTx as DbTransaction); + }, + + /** + * Batched issue insert for company import. + * + * Company import used to call {@link create} once per issue — each call a + * separate network round-trip that inserted the issue, then serialized the + * issue's comments/documents behind the returned id. This inserts a whole + * bundle of pre-resolved issues (ids already generated by the caller) in + * chunked multi-row statements, so a thousand-issue import issues a handful + * of statements instead of thousands. + * + * The per-issue derivations {@link create} performs are reproduced for the + * import subset: a single contiguous identifier range is allocated from the + * company counter, per-project goal/workspace/policy defaults are computed + * once and cached, assignable-agent and workspace validation run per + * distinct id, and monitor notes land un-armed on the row. Dedup, + * idempotency, watchdogs, workspace inheritance and blocked-by wiring — none + * of which import uses — are intentionally omitted. + */ + importIssues: async ( + companyId: string, + rows: ImportIssueRow[], + ): Promise => { + if (rows.length === 0) return; + const isolatedWorkspacesEnabled = ( + await instanceSettings.getExperimental() + ).enableIsolatedWorkspaces; + await db.transaction(async (tx) => { + // Self-correcting counter: seed from max(issue_number) so a drifted + // company counter cannot mint colliding identifiers, then reserve the + // whole range in one bump instead of one-per-issue. + const [maxRow] = await tx + .select({ + maxNum: sql`coalesce(max(${issues.issueNumber}), 0)`, + }) + .from(issues) + .where(eq(issues.companyId, companyId)); + const currentMax = maxRow?.maxNum ?? 0; + const [company] = await tx + .select({ + issueCounter: companies.issueCounter, + issuePrefix: companies.issuePrefix, + }) + .from(companies) + .where(eq(companies.id, companyId)); + if (!company) throw notFound("Target company not found"); + const base = Math.max(company.issueCounter ?? 0, currentMax); + await tx + .update(companies) + .set({ issueCounter: base + rows.length }) + .where(eq(companies.id, companyId)); + + const defaultCompanyGoal = await getDefaultCompanyGoal(tx, companyId); + const defaultGoalId = defaultCompanyGoal?.id ?? null; + + // Project-scoped derivations depend only on the project, so resolve each + // distinct project once and reuse it across that project's issues. + const projectDerivedCache = new Map< + string, + { + goalId: string | null; + defaultProjectWorkspaceId: string | null; + defaultExecutionWorkspaceSettings: Record | null; + } + >(); + const loadProjectDerived = async (projectId: string) => { + const cached = projectDerivedCache.get(projectId); + if (cached) return cached; + const projectRow = await tx + .select({ + goalId: projects.goalId, + executionWorkspacePolicy: projects.executionWorkspacePolicy, + }) + .from(projects) + .where( + and( + eq(projects.id, projectId), + eq(projects.companyId, companyId), + ), + ) + .then((r) => r[0] ?? null); + const policy = parseProjectExecutionWorkspacePolicy( + projectRow?.executionWorkspacePolicy, + ); + const defaultProjectWorkspaceId = await resolveProjectDefaultWorkspaceId(tx, { + companyId, projectId, policy, + }); + const defaultExecutionWorkspaceSettings = + defaultIssueExecutionWorkspaceSettingsForProject( + gateProjectExecutionWorkspacePolicy( + policy, + isolatedWorkspacesEnabled, + ), + ) as Record | null; + const derived = { + goalId: projectRow?.goalId ?? null, + defaultProjectWorkspaceId, + defaultExecutionWorkspaceSettings, + }; + projectDerivedCache.set(projectId, derived); + return derived; + }; + + const validatedAgentIds = new Set(); + const validatedWorkspaceKeys = new Set(); + const issueRows: Array> = []; + const labelRows: Array<{ + issueId: string; + labelId: string; + companyId: string; + }> = []; + + let counter = base; + for (const row of rows) { + await assertExecutionTaskParent(tx as unknown as Db, companyId, row.parentId); + counter += 1; + const issueNumber = counter; + const identifier = `${company.issuePrefix}-${issueNumber}`; + + if (row.assigneeAgentId) { + if (!validatedAgentIds.has(row.assigneeAgentId)) { + await assertAssignableAgent( + tx as unknown as Db, + companyId, + row.assigneeAgentId, + { kind: "work" }, + ); + validatedAgentIds.add(row.assigneeAgentId); + } + } + if (row.status === "in_progress" && !row.assigneeAgentId) { + throw unprocessable("in_progress issues require an assignee"); + } + + const projectId = row.projectId ?? null; + let projectWorkspaceId = row.projectWorkspaceId ?? null; + // Isolated-workspace fields are gated the same way create() gates them: + // when the experiment is off the imported settings are dropped. + let executionWorkspaceSettings = isolatedWorkspacesEnabled + ? (row.executionWorkspaceSettings ?? null) + : null; + let projectGoalId: string | null = null; + if (projectId) { + const derived = await loadProjectDerived(projectId); + projectGoalId = derived.goalId; + if (!projectWorkspaceId) + projectWorkspaceId = derived.defaultProjectWorkspaceId; + if (executionWorkspaceSettings == null) { + executionWorkspaceSettings = + derived.defaultExecutionWorkspaceSettings; + } + } + if (projectWorkspaceId) { + const workspaceKey = `${projectId ?? ""}:${projectWorkspaceId}`; + if (!validatedWorkspaceKeys.has(workspaceKey)) { + await assertValidProjectWorkspace( + companyId, + projectId, + projectWorkspaceId, + tx, + ); + validatedWorkspaceKeys.add(workspaceKey); + } + } + + const goalId = resolveIssueGoalId({ + projectId, + goalId: null, + projectGoalId, + defaultGoalId, + }); + + issueRows.push({ + id: row.id, + companyId, + issueNumber, + identifier, + title: row.title, + description: row.description ?? null, + assigneeAgentId: row.assigneeAgentId ?? null, + status: row.status, + priority: row.priority, + billingCode: row.billingCode ?? null, + assigneeAdapterOverrides: row.assigneeAdapterOverrides ?? null, + projectId, + projectWorkspaceId, + executionWorkspaceSettings, + goalId, + responsibleUserId: null, + requestDepth: clampIssueRequestDepth(undefined), + originKind: "manual", + // The caller resolves parentId against ids in this same batch, so a + // parent always lands in the same insert (rows arrive parents-first). + parentId: row.parentId ?? null, + // Preserved bundle timestamps win; without them createdAt/updatedAt + // fall back to the insert time (the old defaultNow() behavior). + createdAt: row.createdAt ?? new Date(), + updatedAt: row.updatedAt ?? new Date(), + // Imported in-progress work did not start at import time; fabricating + // startedAt here would misrepresent its active episode. + // Only a bundle-carried startedAt is written. + startedAt: row.startedAt ?? null, + completedAt: + row.completedAt ?? (row.status === "done" ? new Date() : null), + cancelledAt: + row.cancelledAt ?? + (row.status === "cancelled" ? new Date() : null), + monitorNotes: row.monitorNotes ?? null, + monitorScheduledBy: row.monitorScheduledBy ?? null, + }); + for (const labelId of new Set(row.labelIds ?? [])) { + labelRows.push({ issueId: row.id, labelId, companyId }); + } + } + + await notifyStatusCardIssueChange(tx, companyId); + await insertRowsInChunks(tx, issues, issueRows); + await insertRowsInChunks(tx, issueLabels, labelRows); + }); + }, + + /** + * Batched comment insert for company import. Comment ids are pre-generated + * by the caller so attachments can reference them without a round-trip. + */ + addImportedComments: async ( + rows: ImportIssueCommentRow[], + ): Promise => { + if (rows.length === 0) return; + const censorUsernameInLogs = (await instanceSettings.getGeneral()) + .censorUsernameInLogs; + await db.transaction(async (tx) => { + for (const companyId of new Set(rows.map(row => row.companyId))) await notifyStatusCardIssueChange(tx, companyId); + const commentRows = rows.map((row) => { + const createdAt = row.createdAt ? new Date(row.createdAt) : null; + return { + id: row.id, + companyId: row.companyId, + issueId: row.issueId, + authorAgentId: row.authorAgentId ?? null, + authorUserId: row.authorUserId ?? null, + authorType: row.authorType, + createdByRunId: null, + body: redactCurrentUserText(row.body, { + enabled: censorUsernameInLogs, + }), + presentation: row.presentation ?? null, + metadata: row.metadata ?? null, + sourceTrust: null, + createdAt: + createdAt && !Number.isNaN(createdAt.getTime()) + ? createdAt + : new Date(), + }; + }); + await insertRowsInChunks(tx, issueComments, commentRows); + // Mirror addComment's recency bump, once per affected issue — but never + // backwards. An issue imported with a preserved updatedAt keeps it + // unless a newer imported comment outdates it; issues without preserved + // timestamps carry an insert-time updatedAt, so GREATEST reproduces the + // old "bump to now" behavior for them. + const bumpAtByIssueId = new Map(); + for (const row of commentRows) { + const existing = bumpAtByIssueId.get(row.issueId); + if (!existing || row.createdAt > existing) + bumpAtByIssueId.set(row.issueId, row.createdAt); + } + const bumpEntries = [...bumpAtByIssueId.entries()]; + for ( + let start = 0; + start < bumpEntries.length; + start += DEFAULT_INSERT_CHUNK_ROWS + ) { + const chunk = bumpEntries.slice( + start, + start + DEFAULT_INSERT_CHUNK_ROWS, + ); + await tx.execute(sql` + update ${issues} + set updated_at = greatest(${issues.updatedAt}, bumps.bump_at) + from (values ${sql.join( + chunk.map( + ([issueId, bumpAt]) => + sql`(${issueId}::uuid, ${bumpAt.toISOString()}::timestamptz)`, + ), + sql`, `, + )}) as bumps(issue_id, bump_at) + where ${issues.id} = bumps.issue_id + `); + } + }); + }, + + /** + * Batched attachment insert for company import: each row mints an asset and + * links it to its issue (and optionally comment) in two chunked statements. + */ + addImportedAttachments: async ( + rows: ImportIssueAttachmentRow[], + ): Promise => { + if (rows.length === 0) return; + await db.transaction(async (tx) => { + const assetRows: Array> = []; + const attachmentRows: Array> = []; + for (const row of rows) { + const assetId = randomUUID(); + assetRows.push({ + id: assetId, + companyId: row.companyId, + provider: row.provider, + objectKey: row.objectKey, + contentType: row.contentType, + byteSize: row.byteSize, + sha256: row.sha256, + originalFilename: row.originalFilename ?? null, + createdByAgentId: row.createdByAgentId ?? null, + createdByUserId: row.createdByUserId ?? null, + }); + attachmentRows.push({ + companyId: row.companyId, + issueId: row.issueId, + assetId, + issueCommentId: row.issueCommentId ?? null, + }); + } + await insertRowsInChunks(tx, assets, assetRows); + await insertRowsInChunks(tx, issueAttachments, attachmentRows); + }); + }, + + update: async ( + id: string, + data: Partial & { + labelIds?: string[]; + blockedByIssueIds?: string[]; + actorAgentId?: string | null; + actorRunId?: string | null; + /** Server-owned transfer being applied; never accepted from HTTP input. */ + caseHandoffId?: string; + actorRunStopId?: string | null; + actorUserId?: string | null; + companyGuard?: string; + workspaceSelectionActor?: AuthorizationActor; + expectedExecutionPolicy?: typeof issues.$inferInsert.executionPolicy; + expectedStatusVersion?: number; + }, + dbOrTx: any = db, + postCommitActivityPublications?: ActivityPublication[], + postCommitActions?: IssuePostCommitAction[], + options: { bindRuntimeSharedWorkspace?: boolean } = {}, + ) => { + const ownedActivityPublications: ActivityPublication[] = []; + const activityPublications = + postCommitActivityPublications ?? ownedActivityPublications; + const ownedPostCommitActions: IssuePostCommitAction[] = []; + const queuedPostCommitActions = + postCommitActions ?? ownedPostCommitActions; + // A caller that supplies `companyGuard` gets the company added to + // every read, lock, and write predicate below. A check before this + // call is not a boundary: `issues.company_id` can change between + // that check and this write, so the predicate must carry the + // company itself. + const idPredicate = + data.companyGuard !== undefined + ? and(eq(issues.id, id), eq(issues.companyId, data.companyGuard)) + : eq(issues.id, id); + const existing = await dbOrTx + .select() + .from(issues) + .where(idPredicate) + .then((rows: Array) => rows[0] ?? null); + if (!existing) return null; + if (data.parentId !== undefined && data.parentId !== existing.parentId) { + await assertExecutionTaskParent(dbOrTx, existing.companyId, data.parentId); + } + if (existing.conversationAgentId) { + if ((data.assigneeAgentId !== undefined && data.assigneeAgentId !== existing.conversationAgentId) + || data.assigneeUserId || data.conversationAgentId !== undefined || data.conversationUserId !== undefined + || data.conversationState !== undefined || data.conversationSessionGeneration !== undefined + || data.conversationBoundaryCommentId !== undefined || data.status === "done" || data.status === "cancelled") { + throw unprocessable("Conversation identity is fixed; finish the reply instead of completing or reassigning the conversation"); + } + } + + const { + labelIds: nextLabelIds, + blockedByIssueIds, + actorAgentId, + actorRunId, + caseHandoffId, + actorRunStopId, + actorUserId, + companyGuard, + workspaceSelectionActor, + expectedExecutionPolicy, + expectedStatusVersion, + ...issueData + } = data; + if (expectedStatusVersion !== undefined && ( + issueData.status !== "todo" || Object.keys(issueData).some(key => key !== "status") || + nextLabelIds !== undefined || blockedByIssueIds !== undefined || expectedExecutionPolicy !== undefined + )) { + throw unprocessable("expectedStatusVersion requires a status-only backlog-to-todo promotion"); + } + // An explicit edit claims the title, even if it keeps the same text. + if (issueData.title !== undefined) issueData.titleNeedsGeneration = false; + if ( + issueData.assigneeAgentId !== undefined && + issueData.assigneeAgentId !== existing.assigneeAgentId + ) { + const externalBinding = await dbOrTx + .select({ id: chatConversations.id }) + .from(chatConversations) + .where( + and( + eq(chatConversations.companyId, existing.companyId), + eq(chatConversations.issueId, existing.id), + ), + ) + .limit(1) + .then((rows: Array<{ id: string }>) => rows[0] ?? null); + if (externalBinding) { + throw conflict( + "Agent assignment cannot change while this task is bound to an external channel", + { + code: "chat_binding_agent_locked", + conversationId: externalBinding.id, + }, + ); + } + } + const isolatedWorkspacesEnabled = ( + await instanceSettings.getExperimental() + ).enableIsolatedWorkspaces; + if (options.bindRuntimeSharedWorkspace) { + const workspaceId = issueData.executionWorkspaceId ?? existing.executionWorkspaceId; + if (!workspaceId) { + throw unprocessable("Runtime workspace binding requires an existing shared workspace"); + } + const [workspace] = await dbOrTx + .select({ mode: executionWorkspaces.mode }) + .from(executionWorkspaces) + .where(and( + eq(executionWorkspaces.id, workspaceId), + eq(executionWorkspaces.companyId, existing.companyId), + )); + if ( + workspace?.mode !== "shared_workspace" || + (issueData.executionWorkspacePreference ?? existing.executionWorkspacePreference) !== "reuse_existing" || + (issueData.executionWorkspaceSettings ?? existing.executionWorkspaceSettings)?.mode !== "shared_workspace" + ) { + throw unprocessable("Runtime workspace binding requires an existing shared workspace"); + } + } + // Warm sandbox continuity is runtime bookkeeping, independent of the + // opt-in UI for creating isolated worktrees. Public updates still obey + // the feature gate; only the internal shared-workspace binding bypasses it. + if (!isolatedWorkspacesEnabled && !options.bindRuntimeSharedWorkspace) { + delete issueData.executionWorkspaceId; + delete issueData.executionWorkspacePreference; + delete issueData.executionWorkspaceSettings; + } + + if (issueData.status) { + assertTransition(existing.status, issueData.status); + } + + const patch: Partial = { + ...issueData, + updatedAt: new Date(), + }; + if (issueData.visibility === "private") { + patch.privacyRootIssueId = existing.privacyRootIssueId ?? existing.id; + } else if (issueData.visibility === "open") { + patch.privacyRootIssueId = null; + } + if (issueData.projectId) { + const selectedProject = await dbOrTx + .select({ visibility: projects.visibility }) + .from(projects) + .where(and(eq(projects.id, issueData.projectId), eq(projects.companyId, existing.companyId))) + .then((rows: Array<{ visibility: string }>) => rows[0] ?? null); + if (!selectedProject) throw notFound("Project not found"); + if (selectedProject.visibility === "private") { + patch.visibility = "private"; + patch.privacyRootIssueId = existing.privacyRootIssueId ?? existing.id; + } + } + if (existing.status !== "blocked" && issueData.status === "blocked") { + patch.blockedTransitionAt = patch.updatedAt; + patch.blockedOwnerNotifiedAt = null; + } else if ( + existing.status === "blocked" && + issueData.status && + issueData.status !== "blocked" + ) { + patch.unblockDescriptor = null; + patch.blockedTransitionAt = null; + patch.blockedOwnerNotifiedAt = null; + } + if (issueData.requestDepth !== undefined) { + patch.requestDepth = clampIssueRequestDepth(issueData.requestDepth); + } + + const nextAssigneeAgentId = + issueData.assigneeAgentId !== undefined + ? issueData.assigneeAgentId + : existing.assigneeAgentId; + const nextAssigneeUserId = + issueData.assigneeUserId !== undefined + ? issueData.assigneeUserId + : existing.assigneeUserId; + + if (nextAssigneeAgentId && nextAssigneeUserId) { + throw unprocessable("Issue can only have one assignee"); + } + if ( + patch.status === "in_progress" && + !nextAssigneeAgentId && + !nextAssigneeUserId + ) { + throw unprocessable("in_progress issues require an assignee"); + } + if (patch.status === "in_progress") { + const dependencyReadiness = + blockedByIssueIds === undefined + ? ( + await listIssueDependencyReadinessMap( + dbOrTx, + existing.companyId, + [id], + ) + ).get(id) + : null; + const unresolvedBlockerIssueIds = + blockedByIssueIds !== undefined + ? await listUnresolvedBlockerIssueIds( + dbOrTx, + existing.companyId, + blockedByIssueIds, + ) + : (dependencyReadiness?.unresolvedBlockerIssueIds ?? []); + if (unresolvedBlockerIssueIds.length > 0) { + const unresolvedBlockers = await listUnresolvedBlockerDetails( + dbOrTx, + existing.companyId, + unresolvedBlockerIssueIds, + dependencyReadiness?.pendingFinalizeBlockerIssueIds, + ); + throw unprocessable("Issue is blocked by unresolved blockers", { + unresolvedBlockerIssueIds, + unresolvedBlockers, + }); + } + } + const shouldValidateNextAssignee = + Boolean(nextAssigneeAgentId) && + (issueData.assigneeAgentId !== undefined || + patch.status === "in_progress"); + if (shouldValidateNextAssignee) { + await assertAssignableAgent( + dbOrTx as Db, + existing.companyId, + nextAssigneeAgentId, + { kind: "work" }, + ); + } + if (issueData.assigneeUserId) { + await assertAssignableUser( + existing.companyId, + issueData.assigneeUserId, + ); + } + let nextProjectId = + issueData.projectId !== undefined + ? issueData.projectId + : existing.projectId; + const nextProjectWorkspaceId = + issueData.projectWorkspaceId !== undefined + ? issueData.projectWorkspaceId + : existing.projectWorkspaceId; + const nextExecutionWorkspaceId = + issueData.executionWorkspaceId !== undefined + ? issueData.executionWorkspaceId + : existing.executionWorkspaceId; + const nextExecutionWorkspacePreference = + issueData.executionWorkspacePreference !== undefined + ? issueData.executionWorkspacePreference + : existing.executionWorkspacePreference; + const nextExecutionWorkspaceSettings = + issueData.executionWorkspaceSettings !== undefined + ? parseIssueExecutionWorkspaceSettings( + issueData.executionWorkspaceSettings, + ) + : parseIssueExecutionWorkspaceSettings( + existing.executionWorkspaceSettings, + ); + if (issueData.executionWorkspaceSettings !== undefined) { + patch.executionWorkspaceSettings = nextExecutionWorkspaceSettings + ? { ...nextExecutionWorkspaceSettings } + : null; + } + if (nextProjectWorkspaceId) await assertValidProjectWorkspace(existing.companyId, null, nextProjectWorkspaceId); + if (nextExecutionWorkspaceId) await assertValidExecutionWorkspace(existing.companyId, null, nextExecutionWorkspaceId); + if ( + isolatedWorkspacesEnabled && + issueData.executionWorkspaceSettings !== undefined + ) { + assertExplicitPinnedWorktreeIssueRunnable({ + projectId: nextProjectId ?? null, + projectWorkspaceId: nextProjectWorkspaceId ?? null, + executionWorkspaceId: nextExecutionWorkspaceId ?? null, + executionWorkspacePreference: + nextExecutionWorkspacePreference ?? null, + executionWorkspaceSettings: issueData.executionWorkspaceSettings, + }); + } + + applyStatusSideEffects(issueData.status, patch); + if (issueData.status && issueData.status !== "done") { + patch.completedAt = null; + } + if (issueData.status && issueData.status !== "cancelled") { + patch.cancelledAt = null; + } + if (issueData.status && issueData.status !== "in_progress") { + patch.checkoutRunId = null; + patch.executionRunId = null; + patch.executionAgentNameKey = null; + patch.executionLockedAt = null; + } + if ( + (issueData.assigneeAgentId !== undefined && + issueData.assigneeAgentId !== existing.assigneeAgentId) || + (issueData.assigneeUserId !== undefined && + issueData.assigneeUserId !== existing.assigneeUserId) + ) { + patch.checkoutRunId = null; + patch.executionRunId = null; + patch.executionAgentNameKey = null; + patch.executionLockedAt = null; + } + + const runUpdate = async (tx: any) => { + const changesPrivacy = issueData.visibility !== undefined || issueData.parentId !== undefined || issueData.projectId !== undefined; + if (changesPrivacy) await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`issue-privacy-tree:${existing.companyId}`}, 0))`); + // The receipt baseline must be read under the same row lock as the + // write. Otherwise a concurrent update can be mistaken for a change + // made by this request. + const receiptExisting = await tx + .select() + .from(issues) + .where(idPredicate) + .for("update") + .then((rows: Array) => rows[0] ?? null); + if (!receiptExisting) return null; + if (issueData.executionPolicy !== undefined) assertQualityReviewPolicyMutation({ previousPolicy: receiptExisting.executionPolicy, nextPolicy: issueData.executionPolicy, actorAgentId, actorUserId }); + if ((patch.status ?? receiptExisting.status) === "done" && + (receiptExisting.status !== "done" || issueData.executionPolicy !== undefined)) { + await assertRequiredQualityReviewComplete(tx as Db, { ...receiptExisting, executionPolicy: issueData.executionPolicy === undefined ? receiptExisting.executionPolicy : issueData.executionPolicy }, true); + } + if (actorRunId) await assertNoCaseHandoffMutation(tx, actorRunId, id); + if (expectedStatusVersion !== undefined && ( + receiptExisting.statusVersion !== expectedStatusVersion || receiptExisting.status !== "backlog" + )) { + throw conflict("The task status or assignment changed. Read the task again before starting work.", { + code: "issue_status_version_changed", + }); + } + if (expectedStatusVersion !== undefined) { + const policy = normalizeIssueExecutionPolicy(receiptExisting.executionPolicy ?? null); + const transition = applyIssueExecutionPolicyTransition({ + issue: receiptExisting, + policy, + previousPolicy: policy, + requestedStatus: "todo", + requestedAssigneePatch: {}, + actor: { agentId: actorAgentId ?? null, userId: actorUserId ?? null }, + allowBoardOverride: workspaceSelectionActor?.type === "board" && workspaceSelectionActor.source !== "mcp_oauth", + }); + if (transition.decision || + (transition.patch.status !== undefined && transition.patch.status !== "todo") || + (transition.patch.assigneeAgentId !== undefined && transition.patch.assigneeAgentId !== receiptExisting.assigneeAgentId) || + (transition.patch.assigneeUserId !== undefined && transition.patch.assigneeUserId !== receiptExisting.assigneeUserId)) { + throw unprocessable("expectedStatusVersion requires a status-only backlog-to-todo promotion"); + } + Object.assign(patch, transition.patch); + } + if (issueData.executionWorkspaceId !== undefined || issueData.projectWorkspaceId !== undefined || + issueData.executionWorkspacePreference !== undefined || issueData.executionWorkspaceSettings !== undefined) { + Object.assign(patch, await executionWorkspaceService(db).prepareTaskWorkspaceUpdate({ + companyId: receiptExisting.companyId, issueId: receiptExisting.id, patch, + }, tx)); + } + if (expectedExecutionPolicy !== undefined && !isDeepStrictEqual( + receiptExisting.executionPolicy ?? null, + expectedExecutionPolicy, + )) { + throw conflict("The task execution settings changed. Try cancelling the monitor again.", { + code: "execution_policy_changed", + }); + } + if (changesPrivacy) { + // Privacy inherited through task ancestry can authorize saved notices + // on descendants as well as this task. + await notifyChatInboundWakeNotices(tx, { companyId: receiptExisting.companyId }); + const nextProjectId = issueData.projectId !== undefined ? issueData.projectId : receiptExisting.projectId; + const [privacyProject] = nextProjectId ? await tx.select().from(projects) + .where(and(eq(projects.id, nextProjectId), eq(projects.companyId, existing.companyId))) : []; + if (privacyProject?.visibility === "private") { + if (issueData.visibility === "open") { + if (!privacyProject.personalOwnerUserId) throw unprocessable("Move this task out of its private project before making it public"); + patch.projectId = null; + } else { + patch.visibility = "private"; + patch.privacyRootIssueId = receiptExisting.privacyRootIssueId ?? receiptExisting.id; + } + } + const privacyParentId = issueData.parentId !== undefined ? issueData.parentId : receiptExisting.privacyParentIssueId; + const [privacyParent] = privacyParentId ? await tx.select().from(issues) + .where(and(eq(issues.id, privacyParentId), eq(issues.companyId, existing.companyId))) : []; + patch.privacyParentIssueId = privacyParent?.id ?? null; + if (privacyParent && (privacyParent.visibility === "private" || (privacyParent.projectId && await tx.select({ id: projects.id }).from(projects) + .where(and(eq(projects.id, privacyParent.projectId), eq(projects.visibility, "private"))).limit(1).then((rows: Array<{ id: string }>) => rows.length > 0)))) { + if (issueData.visibility === "open") throw unprocessable("A task cannot be made public while it inherits private access"); + patch.visibility = "private"; + patch.privacyRootIssueId = privacyParent.privacyRootIssueId ?? privacyParent.id; + } else if (issueData.visibility === "private") { + patch.privacyRootIssueId = receiptExisting.id; + } + } + if (actorAgentId && actorRunId) { + // Recheck under a run lock: a request admitted before Stop must not + // commit a late Done after cancellation revoked its credentials. + const actorRun = await assertAgentRunWriteAllowed(tx, receiptExisting.companyId, { + agentId: actorAgentId, runId: actorRunId, stopId: actorRunStopId, + }); + assertLegacyRunTaskOwnership(actorRun, receiptExisting); + } + if (blockedByIssueIds !== undefined) { + await assertOnboardingFirstTaskDependenciesAllowed(tx as Db, { + companyId: receiptExisting.companyId, + actorAgentId, + actorRunId, + blockedByIssueIds, + parentIssueId: receiptExisting.parentId, + }); + } + if (actorAgentId && patch.status === "done") { + const [review] = await tx.select({ id: toolActionRequests.id }).from(toolActionRequests).where(and(eq(toolActionRequests.companyId, existing.companyId), eq(toolActionRequests.issueId, id), inArray(toolActionRequests.status, ["pending", "approved", "executing"]))).limit(1); + if (review) throw conflict("This task is waiting for a connection review. Finish unrelated work, then yield in_review without retrying the governed call.", { code: "tool_review_pending", actionRequestId: review.id }); + } + + const [previousLabelsByIssueId, previousRelationSummaries] = await Promise.all([ + nextLabelIds !== undefined + ? labelMapForIssues(tx, [id]) + : Promise.resolve(new Map()), + blockedByIssueIds !== undefined + ? getIssueRelationSummaryMap(existing.companyId, [id], tx) + : Promise.resolve(new Map()), + ]); + const defaultCompanyGoal = await getDefaultCompanyGoal(tx, existing.companyId); + const [currentProjectGoalId, nextProjectGoalId] = await Promise.all([ + getProjectDefaultGoalId(tx, existing.companyId, existing.projectId), + getProjectDefaultGoalId( + tx, + existing.companyId, + issueData.projectId !== undefined + ? issueData.projectId + : existing.projectId, + ), + ]); + + patch.goalId = resolveNextIssueGoalId({ + currentProjectId: existing.projectId, + currentGoalId: existing.goalId, + currentProjectGoalId, + projectId: issueData.projectId, + goalId: issueData.goalId, + projectGoalId: nextProjectGoalId, + defaultGoalId: defaultCompanyGoal?.id ?? null, + }); + // Ownership changes invalidate observed handoff versions even if status + // stays the same, including an A -> B -> A assignment race. + if (issueData.status === "done" || issueData.status === "cancelled") { + await cancelCaseHandoffsForIssue(tx, { companyId: receiptExisting.companyId, issueId: id, + reason: `Task was marked ${issueData.status}`, agentTransfersOnly: issueData.status === "done", publications: activityPublications }); + } + if ((issueData.assigneeAgentId !== undefined && issueData.assigneeAgentId !== receiptExisting.assigneeAgentId) + || (issueData.assigneeUserId !== undefined && issueData.assigneeUserId !== receiptExisting.assigneeUserId)) { + await cancelCaseHandoffsForIssue(tx, { companyId: receiptExisting.companyId, issueId: id, + reason: "Task assignment changed", exceptId: caseHandoffId, publications: activityPublications }); + patch.statusVersion = sql`${issues.statusVersion} + 1` as unknown as number; + // Invalidate human direction at the common assignment boundary, including + // plugin/service writes that do not go through HTTP run cancellation. + // Keep the requester attribution for audit; cancellation revokes its use. + await notifyChatInboundWakeNotices(tx, { companyId: receiptExisting.companyId }); + await tx.update(agentWakeupRequests).set({ status: "cancelled", finishedAt: new Date(), updatedAt: new Date() }) + .where(and(eq(agentWakeupRequests.companyId, receiptExisting.companyId), + eq(agentWakeupRequests.requestedByActorType, "user"), + sql`coalesce(${agentWakeupRequests.payload}->>'issueId', ${agentWakeupRequests.payload}->>'taskId', + ${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'issueId', + ${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'taskId') = ${id}`)); + } + // Reasserting Blocked or changing its blockers is a fresh decision even + // when the status string stays the same. Invalidate recovery's prior + // status receipt without treating comment recency as blocking intent. + if (receiptExisting.status === "blocked" && + (issueData.status === "blocked" || + (issueData.status === undefined && + (blockedByIssueIds !== undefined || issueData.unblockDescriptor !== undefined)))) { + patch.statusVersion = sql`${issues.statusVersion} + 1` as unknown as number; + } + await notifyStatusCardIssueChange(tx, receiptExisting.companyId); + if ((patch.status !== undefined && patch.status !== receiptExisting.status) + || patch.completedAt !== undefined || patch.cancelledAt !== undefined + || issueData.parentId !== undefined || issueData.executionWorkspaceId !== undefined) { + await notifyTerminalWorkspaceWork(tx, `issue:${id}`); + } + if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined + || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined) { + await notifyRunDispatch(tx, { companyId: receiptExisting.companyId }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + } + if (patch.monitorNextCheckAt !== undefined || (receiptExisting.monitorNextCheckAt && + (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined))) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + } + if (patch.status === "done" || patch.status === "cancelled") { + await notifyDecisionTargetWork(tx, `issue:${id}`); + } + if ((patch.status !== undefined && patch.status !== receiptExisting.status) || + (patch.assigneeAgentId !== undefined && patch.assigneeAgentId !== receiptExisting.assigneeAgentId) || + (patch.executionState !== undefined && parseObject(receiptExisting.executionState).status === "pending" && + parseObject(patch.executionState).status !== "pending")) { + await notifyChatResponseAvailability(tx, { companyId: receiptExisting.companyId, issueId: id }); + } + if (patch.status !== undefined && patch.status !== receiptExisting.status) { + await notifyChatInboundWakeNotices(tx, { companyId: receiptExisting.companyId, issueId: id }); + } + if (patch.hiddenAt === null && receiptExisting.hiddenAt !== null) { + await notifyChatRunMilestones(tx, { companyId: receiptExisting.companyId, issueId: id }); + } + if (patch.hiddenAt !== undefined || patch.harnessKind !== undefined || patch.conversationAgentId !== undefined) { + await notifyApprovalWork(tx, receiptExisting.companyId); + } + if ((patch.status !== undefined && patch.status !== receiptExisting.status) + || (patch.assigneeAgentId !== undefined && patch.assigneeAgentId !== receiptExisting.assigneeAgentId) + || (patch.executionRunId !== undefined && patch.executionRunId !== receiptExisting.executionRunId) + || (patch.checkoutRunId !== undefined && patch.checkoutRunId !== receiptExisting.checkoutRunId)) { + await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: receiptExisting.companyId, issueId: id }); + } + if (patch.status !== undefined || patch.executionState !== undefined || patch.executionRunId !== undefined + || patch.monitorNextCheckAt !== undefined || patch.hiddenAt !== undefined || patch.originKind !== undefined + || patch.originId !== undefined || patch.conversationAgentId !== undefined || patch.assigneeAgentId !== undefined + || patch.assigneeUserId !== undefined || blockedByIssueIds !== undefined) { + + const affectsDependents = patch.status !== undefined && patch.status !== receiptExisting.status + && (patch.status === "done" || receiptExisting.status === "done"); + await notifyExternalConversationSettlement(tx, { companyId: receiptExisting.companyId, + issueId: affectsDependents ? undefined : id }); + } + if (patch.status !== undefined || patch.statusVersion !== undefined || patch.executionState !== undefined + || patch.executionRunId !== undefined || patch.checkoutRunId !== undefined + || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined || blockedByIssueIds !== undefined) { + await notifyNativeRecoveryNotices(tx, { companyId: receiptExisting.companyId, issueId: id }); + await notifyNativeRunProjection(tx, { companyId: receiptExisting.companyId, issueId: id }); + await notifyNativeResultCommit(tx, { companyId: receiptExisting.companyId, issueId: id }); + await notifyNativeWorkspaceFinalization(tx, { companyId: receiptExisting.companyId, issueId: id }); + await notifyNativeStatusReassessment(tx, { companyId: receiptExisting.companyId, issueId: id }); + } + const updated = await tx + .update(issues) + .set(patch) + .where(idPredicate) + .returning() + .then((rows: Array) => rows[0] ?? null); + if (!updated) return null; + if (updated.status === "done") { + await completeTerminalCaseHandoffsForIssue(tx, { companyId: updated.companyId, issueId: updated.id, + publications: activityPublications }); + } + if (workspaceSelectionActor && updated.workspaceBindingRevision !== receiptExisting.workspaceBindingRevision) { + await executionWorkspaceService(db).assertTaskWorkspaceUpdateAccess({ + task: updated, actor: workspaceSelectionActor, + }, tx); + } + if (issueData.description !== undefined) await attachOwnedDraftImages(tx, updated, updated.description, + { userId: actorUserId, agentId: actorAgentId }); + if (changesPrivacy && updated.visibility === "private") { + // Creation and tree changes hold the same company lock. No child can + // slip into the tree between discovering descendants and protecting them. + const protectedRows = await tx.execute(sql`with recursive descendants as ( + select i.id, i.parent_id, i.privacy_parent_issue_id from issues i + where i.id = ${updated.id} and i.company_id = ${updated.companyId} + union + select i.id, i.parent_id, i.privacy_parent_issue_id from issues i join descendants p + on i.parent_id = p.id or i.privacy_parent_issue_id = p.id + where i.company_id = ${updated.companyId} + ) update issues i set visibility = 'private', + privacy_root_issue_id = ${updated.privacyRootIssueId ?? updated.id}, + privacy_parent_issue_id = coalesce(i.privacy_parent_issue_id, i.parent_id), updated_at = now() + from descendants d where i.id = d.id and i.id <> ${updated.id} returning i.id`); + if (protectedRows.length) { + const descendants = await tx.select().from(issues).where(inArray(issues.id, protectedRows.map((row: { id: string }) => row.id))); + for (const descendant of descendants) await ensureAssignmentIssueAccessGrant(tx, descendant, null, { + agentId: actorAgentId ?? null, userId: actorUserId ?? null, + }); + } + } + await ensureAssignmentIssueAccessGrant(tx, updated, receiptExisting, { + agentId: actorAgentId ?? null, + userId: actorUserId ?? null, + }); + await recordChatCompletion(tx, receiptExisting, updated); + // An operator explicitly choosing a disposition owns that decision, + // including choosing In Review while the conversation is Idle. + if (actorUserId && issueData.status !== undefined) { + await tx.update(chatConversations).set({ state: "active", updatedAt: new Date() }) + .where(and(eq(chatConversations.companyId, updated.companyId), eq(chatConversations.issueId, updated.id), + eq(chatConversations.state, "waiting"), sql`exists (select 1 from chat_endpoints e + where e.id = ${chatConversations.endpointId} and e.company_id = ${chatConversations.companyId} and e.provider = 'slack')`)); + } + if (updated.assigneeAgentId !== existing.assigneeAgentId || updated.assigneeUserId !== existing.assigneeUserId) { + const { issueThreadInteractionService } = await import("./issue-thread-interactions.js"); + await issueThreadInteractionService(tx).expireConnectionIntentsForOwnershipChange(updated); + } + if (existing.status !== updated.status) { + if ( + (existing.status === "done" || existing.status === "cancelled") && + updated.status !== "done" && + updated.status !== "cancelled" + ) { + const terminalWorkspaces = await tx + .select({ id: executionWorkspaces.id }) + .from(executionWorkspaces) + .where( + and( + eq(executionWorkspaces.companyId, updated.companyId), + eq(executionWorkspaces.sourceIssueId, updated.id), + eq(executionWorkspaces.status, "archived"), + like(executionWorkspaces.cleanupReason, "issue_terminal%"), + ), + ); + for (const workspace of terminalWorkspaces) { + await logActivity(tx as unknown as Db, { + companyId: updated.companyId, + actorType: actorAgentId + ? "agent" + : actorUserId + ? "user" + : "system", + actorId: actorAgentId ?? actorUserId ?? "issue_service", + agentId: actorAgentId ?? null, + action: "execution_workspace.source_issue_reopened", + entityType: "execution_workspace", + entityId: workspace.id, + details: { + sourceIssueId: updated.id, + previousIssueStatus: existing.status, + nextIssueStatus: updated.status, + workspaceAction: "left_archived", + }, + }); + } + } + if (updated.status === "done" || updated.status === "cancelled") { + await finalizeSummarySlotsForTerminalIssue(tx, updated); + // Every terminal transition funnels through here, including direct + // service callers (tree control, recovery, status cards) + // that never touch the HTTP routes. Governed cards expire; ordinary + // historical questions remain answerable after completion. + // Dynamic import breaks the module + // cycle (issue-thread-interactions.js imports issueService). + const { issueThreadInteractionService } = + await import("./issue-thread-interactions.js"); + // Stop live question requests independently of card expiry. A + // historical question retained in the feed must not keep its + // source run waiting after the task closes. + const pendingQuestions = await tx + .select() + .from(issueThreadInteractions) + .where(and( + eq(issueThreadInteractions.companyId, updated.companyId), + eq(issueThreadInteractions.issueId, updated.id), + eq(issueThreadInteractions.kind, "ask_user_questions"), + eq(issueThreadInteractions.status, "pending"), + )); + const expiredInteractions = await issueThreadInteractionService( + tx, + ).expirePendingInteractionsForTerminalIssue(updated, { + agentId: actorAgentId ?? null, + userId: actorUserId ?? null, + }); + const { + nativeQuestionCancellationIdentity, + requestNativeQuestionRunCancellation, + } = await import("./native-runtime/native-question-bridge.js"); + for (const interaction of pendingQuestions) { + const nativeQuestion = + nativeQuestionCancellationIdentity(interaction); + if (nativeQuestion) { + if (dbOrTx !== db && !postCommitActions) { + throw new Error( + "Terminal native question updates in an external transaction require a post-commit action queue", + ); + } + const runId = await requestNativeQuestionRunCancellation( + tx, + nativeQuestion, + { kind: "issue_terminal", issueStatus: updated.status }, + ); + if (runId) { + queuedPostCommitActions.push({ + type: "cancel_native_question_run", + runId, + issueId: updated.id, + issueStatus: updated.status, + }); + } + } + } + for (const interaction of expiredInteractions) { + await logActivity(tx as unknown as Db, { + companyId: updated.companyId, + actorType: actorAgentId + ? "agent" + : actorUserId + ? "user" + : "system", + actorId: actorAgentId ?? actorUserId ?? "issue_service", + agentId: actorAgentId ?? null, + action: "issue.thread_interaction_expired", + entityType: "issue", + entityId: updated.id, + details: { + identifier: updated.identifier ?? null, + interactionId: interaction.id, + interactionKind: interaction.kind, + interactionStatus: interaction.status, + source: "issue.status_transition.issue_closed", + result: interaction.result ?? null, + }, + }); + } + } + // A status-card generation task that goes done/cancelled/blocked stops + // making progress; release the card's generation claim so the board tile + // stops spinning and offers "Run now" again (blocked = stuck on a human). + if ( + updated.status === "done" || + updated.status === "cancelled" || + updated.status === "blocked" + ) { + await finalizeStatusCardsForStalledGeneration(tx, updated); + } + } + if (nextLabelIds !== undefined) { + await syncIssueLabels( + updated.id, + existing.companyId, + nextLabelIds, + tx, + ); + } + if (blockedByIssueIds !== undefined) { + await syncBlockedByIssueIds( + updated.id, + existing.companyId, + blockedByIssueIds, + { + agentId: actorAgentId ?? null, + userId: actorUserId ?? null, + }, + tx, + ); + } + if ( + issueData.executionWorkspaceSettings !== undefined && + nextExecutionWorkspaceId && + nextExecutionWorkspacePreference === "reuse_existing" + ) { + const workspace = await tx + .select({ + id: executionWorkspaces.id, + metadata: executionWorkspaces.metadata, + }) + .from(executionWorkspaces) + .where( + and( + eq(executionWorkspaces.id, nextExecutionWorkspaceId), + eq(executionWorkspaces.companyId, existing.companyId), + ), + ) + .then( + (rows: Array<{ id: string; metadata: unknown }>) => + rows[0] ?? null, + ); + if (workspace) { + await tx + .update(executionWorkspaces) + .set({ + metadata: mergeExecutionWorkspaceConfig( + (workspace.metadata as Record | null) ?? + null, + buildReusedExecutionWorkspaceConfigPatchFromIssueSettings( + nextExecutionWorkspaceSettings, + ), + ), + updatedAt: new Date(), + }) + .where(eq(executionWorkspaces.id, workspace.id)); + } + } + const [enriched] = await withIssueLabels(tx, [updated]); + const nextBlockedByIssueIds = + blockedByIssueIds === undefined + ? undefined + : [...new Set(blockedByIssueIds)].sort(); + const changes = buildIssueChanges( + receiptExisting as unknown as Record, + updated as unknown as Record, + { + ...(nextLabelIds !== undefined + ? { + labelIds: { + from: (previousLabelsByIssueId.get(id) ?? []).map( + (label) => label.id, + ), + to: enriched.labelIds, + }, + } + : {}), + ...(nextBlockedByIssueIds !== undefined + ? { + blockedByIssueIds: { + from: ( + previousRelationSummaries.get(id)?.blockedBy ?? [] + ).map((relation) => relation.id), + to: nextBlockedByIssueIds, + }, + } + : {}), + }, + ); + if ( + (issueData.status === "done" || issueData.status === "cancelled") && + existing.status !== issueData.status && + existing.originKind === + RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation + ) { + const parsedIncident = parseIssueGraphLivenessIncidentKey( + existing.originId, + ); + if ( + parsedIncident?.issueId && + parsedIncident.companyId === existing.companyId + ) { + await notifyExternalConversationSettlement(tx, { companyId: existing.companyId, issueId: parsedIncident.issueId }); + await tx + .delete(issueRelations) + .where( + and( + eq(issueRelations.companyId, existing.companyId), + eq(issueRelations.issueId, existing.id), + eq(issueRelations.relatedIssueId, parsedIncident.issueId), + eq(issueRelations.type, "blocks"), + ), + ); + } + } + if ( + actorUserId && + receiptExisting.status !== "done" && + updated.status === "done" + ) { + if (dbOrTx !== db && !postCommitActivityPublications) { + throw new Error( + "Human completion in an external transaction requires a post-commit activity queue", + ); + } + const now = new Date(); + const archiveState = await archiveInbox( + updated.companyId, + updated.id, + actorUserId, + now, + undefined, + tx, + ); + const { publication } = await persistActivity(tx as unknown as Db, { + companyId: updated.companyId, + actorType: "user", + actorId: actorUserId, + action: "issue.inbox_archived", + entityType: "issue", + entityId: updated.id, + details: { + userId: actorUserId, + archivedAt: archiveState.archivedAt, + targetResolvedFrom: "responsible_user", + source: "issue_status_done", + }, + }); + activityPublications.push(publication); + } + return { + ...enriched, + ...(nextBlockedByIssueIds !== undefined + ? { blockedByIssueIds: nextBlockedByIssueIds } + : {}), + changes, + }; + }; + + const result = await (dbOrTx === db + ? db.transaction(runUpdate) + : runUpdate(dbOrTx)); + if (dbOrTx === db && !postCommitActivityPublications) { + for (const publication of ownedActivityPublications) + publishActivity(publication); + } + if (dbOrTx === db && !postCommitActions) { + await executeIssuePostCommitActions(db, ownedPostCommitActions); + } + return result; + }, + + clearExecutionWorkspaceEnvironmentSelection: async ( + companyId: string, + environmentId: string, + ) => { + const rows = await db + .select({ + id: issues.id, + executionWorkspaceSettings: issues.executionWorkspaceSettings, + }) + .from(issues) + .where(eq(issues.companyId, companyId)); + + let cleared = 0; + for (const row of rows) { + const settings = parseIssueExecutionWorkspaceSettings( + row.executionWorkspaceSettings, + { includeEnvironmentId: true }, + ); + if (settings?.environmentId !== environmentId) continue; + + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, companyId); + await tx + .update(issues) + .set({ + executionWorkspaceSettings: { + ...settings, + environmentId: null, + }, + updatedAt: new Date(), + }) + .where(eq(issues.id, row.id)); + }); + cleared += 1; + } + + return cleared; + }, + + remove: (id: string) => + db.transaction(async (tx) => { + const attachmentAssetIds = await tx + .select({ assetId: issueAttachments.assetId }) + .from(issueAttachments) + .where(eq(issueAttachments.issueId, id)); + const issueDocumentIds = await tx + .select({ documentId: issueDocuments.documentId }) + .from(issueDocuments) + .where(eq(issueDocuments.issueId, id)); + + let removedIssue; + try { + await notifyTerminalWorkspaceWork(tx, `issue:${id}`); + await requestRestorePointDeletion(tx, { sourceIssueId: id }); + const [owner] = await tx.select({ companyId: issues.companyId }).from(issues).where(eq(issues.id, id)); + if (owner) { + await notifyExternalConversationSettlement(tx, { companyId: owner.companyId }); + await notifyApprovalWork(tx, owner.companyId); + await notifyStatusCardIssueChange(tx, owner.companyId); + } + await notifyDecisionTargetIssueRemoval(tx, id); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx, owner ? { companyId: owner.companyId } : undefined); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + removedIssue = await tx + .delete(issues) + .where(eq(issues.id, id)) + .returning() + .then((rows) => rows[0] ?? null); + } catch (err) { + // A foreign key to issues.id without a delete policy blocks the delete + // and raises SQLSTATE 23503. Map it to a clear 409 instead of a bare + // 500. This also covers the decisions table, whose NOT NULL references + // to issues.id stay restricted on purpose. + if (isForeignKeyViolation(err)) { + throw conflict( + "Issue cannot be deleted because another record still references it.", + ); + } + throw err; + } + + if (removedIssue && attachmentAssetIds.length > 0) { + await tx.delete(assets).where( + inArray( + assets.id, + attachmentAssetIds.map((row) => row.assetId), + ), + ); + } + + if (removedIssue && issueDocumentIds.length > 0) { + await tx.delete(documents).where( + inArray( + documents.id, + issueDocumentIds.map((row) => row.documentId), + ), + ); + } + + if (!removedIssue) return null; + const [enriched] = await withIssueLabels(tx, [removedIssue]); + return enriched; + }), + + checkout: async ( + id: string, + agentId: string, + expectedStatuses: string[], + checkoutRunId: string | null, + ) => { + const issueCompany = await db + .select({ companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, id)) + .then((rows) => rows[0] ?? null); + if (!issueCompany) throw notFound("Issue not found"); + await assertAssignableAgent(db, issueCompany.companyId, agentId, { + kind: "work", + }); + + const now = new Date(); + const activePauseHold = await treeControlSvc.getActivePauseHoldGate( + issueCompany.companyId, + id, + ); + if ( + activePauseHold && + !(await isTreeHoldInteractionCheckoutAllowed( + issueCompany.companyId, + checkoutRunId, + activePauseHold, + )) + ) { + throw conflict("Issue checkout blocked by active subtree pause hold", { + issueId: id, + holdId: activePauseHold.holdId, + rootIssueId: activePauseHold.rootIssueId, + mode: activePauseHold.mode, + securityPrinciples: [ + "Complete Mediation", + "Fail Securely", + "Secure Defaults", + ], + }); + } + + await clearExecutionRunIfTerminal(id); + await clearCheckoutRunIfTerminal(id); + + const dependencyReadiness = await listIssueDependencyReadinessMap( + db, + issueCompany.companyId, + [id], + ); + const readiness = dependencyReadiness.get(id); + const unresolvedBlockerIssueIds = + readiness?.unresolvedBlockerIssueIds ?? []; + if (unresolvedBlockerIssueIds.length > 0) { + const unresolvedBlockers = await listUnresolvedBlockerDetails( + db, + issueCompany.companyId, + unresolvedBlockerIssueIds, + readiness?.pendingFinalizeBlockerIssueIds, + ); + throw unprocessable("Issue is blocked by unresolved blockers", { + unresolvedBlockerIssueIds, + unresolvedBlockers, + }); + } + + const sameRunAssigneeCondition = checkoutRunId + ? and( + eq(issues.assigneeAgentId, agentId), + or( + isNull(issues.checkoutRunId), + eq(issues.checkoutRunId, checkoutRunId), + ), + ) + : and( + eq(issues.assigneeAgentId, agentId), + isNull(issues.checkoutRunId), + ); + const executionLockCondition = checkoutRunId + ? or( + isNull(issues.executionRunId), + eq(issues.executionRunId, checkoutRunId), + ) + : isNull(issues.executionRunId); + const updated = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, issueCompany.companyId); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx, { companyId: issueCompany.companyId }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + return tx + .update(issues) + .set({ + assigneeAgentId: agentId, + assigneeUserId: null, + checkoutRunId, + executionRunId: checkoutRunId, + status: "in_progress", + startedAt: now, + updatedAt: now, + }) + .where( + and( + eq(issues.id, id), + inArray(issues.status, expectedStatuses), + or(isNull(issues.assigneeAgentId), sameRunAssigneeCondition), + executionLockCondition, + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + }); + + if (updated) { + const [enriched] = await withIssueLabels(db, [updated]); + return enriched; + } + + const current = await db + .select({ + id: issues.id, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + }) + .from(issues) + .where(eq(issues.id, id)) + .then((rows) => rows[0] ?? null); + + if (!current) throw notFound("Issue not found"); + + if ( + current.assigneeAgentId === agentId && + current.status === "in_progress" && + current.checkoutRunId == null && + (current.executionRunId == null || + current.executionRunId === checkoutRunId) && + checkoutRunId + ) { + const adopted = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, issueCompany.companyId); + return tx + .update(issues) + .set({ + checkoutRunId, + executionRunId: checkoutRunId, + updatedAt: new Date(), + }) + .where( + and( + eq(issues.id, id), + eq(issues.status, "in_progress"), + eq(issues.assigneeAgentId, agentId), + isNull(issues.checkoutRunId), + or( + isNull(issues.executionRunId), + eq(issues.executionRunId, checkoutRunId), + ), + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + }); + if (adopted) return adopted; + } + + if ( + checkoutRunId && + current.assigneeAgentId === agentId && + current.status === "in_progress" && + current.checkoutRunId && + current.checkoutRunId !== checkoutRunId + ) { + const staleAdoption = await adoptStaleCheckoutRun({ + issueId: id, + actorAgentId: agentId, + actorRunId: checkoutRunId, + expectedCheckoutRunId: current.checkoutRunId, + }); + if (staleAdoption.adopted) { + const row = await db + .select() + .from(issues) + .where(eq(issues.id, id)) + .then((rows) => rows[0] ?? null); + if (!row) throw notFound("Issue not found"); + const [enriched] = await withIssueLabels(db, [row]); + return enriched; + } + } + + // Adopt stale executionRunId — if the execution lock points to a terminal/missing run, clear it and proceed. + // Only adopts when the caller's expectedStatuses guard still holds; preserves any existing assigneeUserId + // and preserves the original startedAt when the issue is already in_progress. + if ( + checkoutRunId && + current.executionRunId && + current.executionRunId !== checkoutRunId && + (current.assigneeAgentId === agentId || current.assigneeAgentId == null) + ) { + const executionRun = await db.select().from(heartbeatRuns) + .where(eq(heartbeatRuns.id, current.executionRunId)).then(rows => rows[0] ?? null); + const stale = !executionRun || TERMINAL_HEARTBEAT_RUN_STATUSES.has(executionRun.status); + if (stale && !isExplicitContinuationRetryClaim({ ...current, companyId: issueCompany.companyId }, executionRun)) { + const now = new Date(); + const adoptionSet: Record = { + assigneeAgentId: agentId, + checkoutRunId, + executionRunId: checkoutRunId, + executionAgentNameKey: null, + executionLockedAt: now, + status: "in_progress", + updatedAt: now, + }; + if (current.status !== "in_progress") { + adoptionSet.startedAt = now; + } + const staleExecutionRunId = current.executionRunId; + const adopted = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, issueCompany.companyId); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx, { companyId: issueCompany.companyId }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + return tx + .update(issues) + .set(adoptionSet) + .where( + and( + eq(issues.id, id), + inArray(issues.status, expectedStatuses), + eq(issues.executionRunId, staleExecutionRunId), + or( + isNull(issues.assigneeAgentId), + eq(issues.assigneeAgentId, agentId), + ), + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + }); + if (adopted) { + const [enriched] = await withIssueLabels(db, [adopted]); + return enriched; + } + } + } + + // If this run already owns it and it's in_progress, return it (no self-409) + if ( + current.assigneeAgentId === agentId && + current.status === "in_progress" && + sameRunLock(current.checkoutRunId, checkoutRunId) + ) { + const row = await db + .select() + .from(issues) + .where(eq(issues.id, id)) + .then((rows) => rows[0] ?? null); + if (!row) throw notFound("Issue not found"); + const [enriched] = await withIssueLabels(db, [row]); + return enriched; + } + + throw conflict("Issue checkout conflict", { + issueId: current.id, + status: current.status, + assigneeAgentId: current.assigneeAgentId, + checkoutRunId: current.checkoutRunId, + executionRunId: current.executionRunId, + }); + }, + + assertCheckoutOwner: async ( + id: string, + actorAgentId: string, + actorRunId: string | null, + ) => { + await clearExecutionRunIfTerminal(id); + await clearCheckoutRunIfTerminal(id); + const loadCurrent = () => + db + .select({ + id: issues.id, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + }) + .from(issues) + .where(eq(issues.id, id)) + .then((rows) => rows[0] ?? null); + const current = await loadCurrent(); + + if (!current) throw notFound("Issue not found"); + + const resolveSameRunOwnership = (candidate: { + id: string; + status: string; + assigneeAgentId: string | null; + checkoutRunId: string | null; + executionRunId: string | null; + }) => { + if ( + candidate.status === "in_progress" && + candidate.assigneeAgentId === actorAgentId && + sameRunLock(candidate.checkoutRunId, actorRunId) + ) { + return { ...candidate, adoptedFromRunId: null as string | null }; + } + return null; + }; + + const canAdoptUnownedCheckout = (candidate: { + status: string; + assigneeAgentId: string | null; + checkoutRunId: string | null; + executionRunId: string | null; + }) => + actorRunId && + candidate.status === "in_progress" && + candidate.assigneeAgentId === actorAgentId && + candidate.checkoutRunId == null && + (candidate.executionRunId == null || + candidate.executionRunId === actorRunId); + + const resolveOwnership = async (candidate: { + id: string; + status: string; + assigneeAgentId: string | null; + checkoutRunId: string | null; + executionRunId: string | null; + }) => { + const sameRunOwnership = resolveSameRunOwnership(candidate); + if (sameRunOwnership) + return { ownership: sameRunOwnership, latest: null }; + + if (canAdoptUnownedCheckout(candidate)) { + const adopted = await adoptUnownedCheckoutRun({ + issueId: id, + actorAgentId, + actorRunId: actorRunId!, + }); + + if (adopted) { + return { + ownership: { + ...adopted, + adoptedFromRunId: null as string | null, + }, + latest: null, + }; + } + } + + if ( + actorRunId && + candidate.status === "in_progress" && + candidate.assigneeAgentId === actorAgentId && + candidate.checkoutRunId && + candidate.checkoutRunId !== actorRunId + ) { + const previousCheckoutRunId = candidate.checkoutRunId; + const staleAdoption = await adoptStaleCheckoutRun({ + issueId: id, + actorAgentId, + actorRunId, + expectedCheckoutRunId: previousCheckoutRunId, + }); + + if (staleAdoption.adopted) { + return { + ownership: { + ...staleAdoption.adopted, + adoptedFromRunId: previousCheckoutRunId, + }, + latest: null, + }; + } + + if (staleAdoption.latest) { + const latestOwnership = resolveSameRunOwnership( + staleAdoption.latest, + ); + if (latestOwnership) + return { + ownership: latestOwnership, + latest: staleAdoption.latest, + }; + return { ownership: null, latest: staleAdoption.latest }; + } + } + + return { ownership: null, latest: null }; + }; + + const resolved = await resolveOwnership(current); + if (resolved.ownership) return resolved.ownership; + + const latest = resolved.latest ?? (await loadCurrent()); + if (!latest) throw notFound("Issue not found"); + const resolvedLatest = await resolveOwnership(latest); + if (resolvedLatest.ownership) return resolvedLatest.ownership; + if (resolvedLatest.latest) { + throw conflict("Issue run ownership conflict", { + issueId: resolvedLatest.latest.id, + status: resolvedLatest.latest.status, + assigneeAgentId: resolvedLatest.latest.assigneeAgentId, + checkoutRunId: resolvedLatest.latest.checkoutRunId, + executionRunId: resolvedLatest.latest.executionRunId, + actorAgentId, + actorRunId, + }); + } + + throw conflict("Issue run ownership conflict", { + issueId: latest.id, + status: latest.status, + assigneeAgentId: latest.assigneeAgentId, + checkoutRunId: latest.checkoutRunId, + executionRunId: latest.executionRunId, + actorAgentId, + actorRunId, + }); + }, + + release: async ( + id: string, + actorAgentId?: string, + actorRunId?: string | null, + ) => + db.transaction(async (tx) => { + await tx.execute( + sql`select ${issues.id} from ${issues} where ${issues.id} = ${id} for update`, + ); + const existing = await tx + .select() + .from(issues) + .where(eq(issues.id, id)) + .then((rows) => rows[0] ?? null); + + if (!existing) return null; + if ( + actorAgentId && + existing.assigneeAgentId && + existing.assigneeAgentId !== actorAgentId + ) { + throw conflict("Only assignee can release issue"); + } + if ( + actorAgentId && + existing.status === "in_progress" && + existing.assigneeAgentId === actorAgentId && + existing.checkoutRunId && + !sameRunLock(existing.checkoutRunId, actorRunId ?? null) + ) { + const stale = await isTerminalOrMissingHeartbeatRun( + existing.checkoutRunId, + tx, + ); + if (!stale) { + throw conflict("Only checkout run can release issue", { + issueId: existing.id, + assigneeAgentId: existing.assigneeAgentId, + checkoutRunId: existing.checkoutRunId, + actorRunId: actorRunId ?? null, + }); + } + } + + // Terminal assignment records who owned the work, not a live execution + // claim. Cleanup must preserve it; unfinished release still relinquishes it. + const isTerminal = existing.status === "done" || existing.status === "cancelled"; + const releaseStatus = + existing.status === "in_progress" ? "todo" : existing.status; + await notifyTerminalWorkspaceWork(tx, `issue:${id}`); + await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx, { companyId: existing.companyId }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + const updated = await tx + .update(issues) + .set({ + status: releaseStatus, + assigneeAgentId: isTerminal ? existing.assigneeAgentId : null, + checkoutRunId: null, + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: new Date(), + }) + .where(eq(issues.id, id)) + .returning() + .then((rows) => rows[0] ?? null); + if (!updated) return null; + const [enriched] = await withIssueLabels(tx, [updated]); + return enriched; + }), + + adminForceRelease: async ( + id: string, + options: { clearAssignee?: boolean } = {}, + ) => + db.transaction(async (tx) => { + await tx.execute( + sql`select ${issues.id} from ${issues} where ${issues.id} = ${id} for update`, + ); + const existing = await tx + .select({ + id: issues.id, + companyId: issues.companyId, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + }) + .from(issues) + .where(eq(issues.id, id)) + .then((rows) => rows[0] ?? null); + if (!existing) return null; + + const patch: Partial = { + checkoutRunId: null, + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: new Date(), + }; + if (options.clearAssignee) { + patch.assigneeAgentId = null; + } + + await notifyStatusCardIssueChange(tx, existing.companyId); + await notifyTerminalWorkspaceWork(tx, `issue:${id}`); + if (existing.checkoutRunId || existing.executionRunId || options.clearAssignee) { + await notifyExternalConversationSettlement(tx, { companyId: existing.companyId, issueId: id }); + await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id }); + } + if (existing.checkoutRunId || existing.executionRunId) { + await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: existing.companyId, issueId: id }); + } + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx, { companyId: existing.companyId }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + const updated = await tx + .update(issues) + .set(patch) + .where(eq(issues.id, id)) + .returning() + .then((rows) => rows[0] ?? null); + if (!updated) return null; + + const [enriched] = await withIssueLabels(tx, [updated]); + return { + issue: enriched, + previous: { + checkoutRunId: existing.checkoutRunId, + executionRunId: existing.executionRunId, + }, + }; + }), + + listLabels: (companyId: string) => + db + .select() + .from(labels) + .where(eq(labels.companyId, companyId)) + .orderBy(asc(labels.name), asc(labels.id)), + + getLabelById: (id: string) => + db + .select() + .from(labels) + .where(eq(labels.id, id)) + .then((rows) => rows[0] ?? null), + + createLabel: async ( + companyId: string, + data: Pick, + ) => { + const [created] = await db + .insert(labels) + .values({ + companyId, + name: data.name.trim(), + color: data.color, + }) + .returning(); + return created; + }, + + deleteLabel: async (id: string) => db.transaction(async tx => { + const [label] = await tx.select({ companyId: labels.companyId }).from(labels).where(eq(labels.id, id)); + if (!label) return null; + await notifyStatusCardIssueChange(tx, label.companyId); + return tx.delete(labels).where(eq(labels.id, id)).returning().then(rows => rows[0] ?? null); + }), + + listComments: async ( + issueId: string, + opts?: { + afterCommentId?: string | null; + order?: "asc" | "desc"; + limit?: number | null; + humanOnly?: boolean; + companyId?: string; + }, + ) => { + const order = opts?.order === "asc" ? "asc" : "desc"; + const afterCommentId = opts?.afterCommentId?.trim() || null; + const limit = + opts?.limit && opts.limit > 0 + ? Math.min(Math.floor(opts.limit), MAX_ISSUE_COMMENT_PAGE_LIMIT) + : null; + + const conditions = [eq(issueComments.issueId, issueId)]; + if (opts?.companyId) conditions.push(eq(issueComments.companyId, opts.companyId)); + if (opts?.humanOnly) conditions.push(or( + eq(issueComments.authorType, "user"), + and(isNull(issueComments.authorType), isNotNull(issueComments.authorUserId)), + )!); + if (afterCommentId) { + // Guard: reject non-UUID cursors before hitting the DB to avoid Postgres type errors. + if (!isUuidLike(afterCommentId)) return []; + const anchor = await db + .select({ + id: issueComments.id, + createdAt: issueComments.createdAt, + }) + .from(issueComments) + .where( + and( + eq(issueComments.issueId, issueId), + eq(issueComments.id, afterCommentId), + ), + ) + .then((rows) => rows[0] ?? null); + + if (!anchor) return []; + const anchorCreatedAt = + anchor.createdAt instanceof Date + ? anchor.createdAt + : new Date(String(anchor.createdAt)); + conditions.push( + order === "asc" + ? or( + gt(issueComments.createdAt, anchorCreatedAt), + and( + eq(issueComments.createdAt, anchorCreatedAt), + gt(issueComments.id, anchor.id), + ), + )! + : or( + lt(issueComments.createdAt, anchorCreatedAt), + and( + eq(issueComments.createdAt, anchorCreatedAt), + lt(issueComments.id, anchor.id), + ), + )!, + ); + } + + const query = db + .select() + .from(issueComments) + .where(and(...conditions)) + .orderBy( + order === "asc" + ? asc(issueComments.createdAt) + : desc(issueComments.createdAt), + order === "asc" ? asc(issueComments.id) : desc(issueComments.id), + ); + + const comments = limit ? await query.limit(limit) : await query; + const { censorUsernameInLogs } = await instanceSettings.getGeneral(); + const projectedComments = await projectHistoricalRunComments(comments); + const enrichedComments = + await enrichCommentsWithDerivedAgentAttribution(projectedComments); + return enrichedComments.map((comment) => + redactIssueComment(comment, censorUsernameInLogs), + ); + }, + + getCommentCursor: async (issueId: string) => { + const [latest, countRow] = await Promise.all([ + db + .select({ + latestCommentId: issueComments.id, + latestCommentAt: issueComments.createdAt, + }) + .from(issueComments) + .where(eq(issueComments.issueId, issueId)) + .orderBy(desc(issueComments.createdAt), desc(issueComments.id)) + .limit(1) + .then((rows) => rows[0] ?? null), + db + .select({ + totalComments: sql`count(*)::int`, + }) + .from(issueComments) + .where(eq(issueComments.issueId, issueId)) + .then((rows) => rows[0] ?? null), + ]); + + return { + totalComments: Number(countRow?.totalComments ?? 0), + latestCommentId: latest?.latestCommentId ?? null, + latestCommentAt: latest?.latestCommentAt ?? null, + }; + }, + + getComment: async (commentId: string) => { + const { censorUsernameInLogs } = await instanceSettings.getGeneral(); + const comment = await db + .select() + .from(issueComments) + .where(eq(issueComments.id, commentId)) + .then((rows) => rows[0] ?? null); + if (!comment) return null; + const [projectedComment] = await projectHistoricalRunComments([comment]); + const [enrichedComment] = await enrichCommentsWithDerivedAgentAttribution( + [projectedComment ?? comment], + ); + return redactIssueComment( + enrichedComment ?? projectedComment ?? comment, + censorUsernameInLogs, + ); + }, + + removeComment: async (commentId: string) => { + const currentUserRedactionOptions = { + enabled: (await instanceSettings.getGeneral()).censorUsernameInLogs, + }; + + return db.transaction(async (tx) => { + if (!(await lockIssueForCommentMutation(tx, commentId))) return null; + const [source] = await tx.select({ companyId: issueComments.companyId, issueId: issueComments.issueId }) + .from(issueComments).where(eq(issueComments.id, commentId)); + if (!source) return null; + await notifyExternalConversationSettlement(tx, source); + await notifyStatusCardIssueChange(tx, source.companyId); + const [comment] = await tx + .delete(issueComments) + .where(eq(issueComments.id, commentId)) + .returning(); + + if (!comment) return null; + + await tx + .update(issues) + .set({ updatedAt: new Date() }) + .where(eq(issues.id, comment.issueId)); + + return redactIssueComment(comment, currentUserRedactionOptions.enabled); + }); + }, + + tombstoneComment: async ( + commentId: string, + actor: { + actorType: "agent" | "user"; + agentId?: string | null; + userId?: string | null; + runId?: string | null; + }, + options?: { + afterTombstone?: (comment: IssueComment, tx: any) => Promise; + }, + ) => { + const currentUserRedactionOptions = { + enabled: (await instanceSettings.getGeneral()).censorUsernameInLogs, + }; + + return db.transaction(async (tx) => { + if (!(await lockIssueForCommentMutation(tx, commentId))) return null; + const [source] = await tx.select({ companyId: issueComments.companyId, issueId: issueComments.issueId }) + .from(issueComments).where(and(eq(issueComments.id, commentId), isNull(issueComments.deletedAt))) + .for("update"); + if (!source) return null; + await notifyExternalConversationSettlement(tx, source); + await notifyStatusCardIssueChange(tx, source.companyId); + await notifyChatInboundWakeNotices(tx, source); + const now = new Date(); + const [comment] = await tx + .update(issueComments) + .set({ + body: DELETED_ISSUE_COMMENT_BODY, + presentation: null, + metadata: null, + deletedAt: now, + deletedByType: actor.actorType, + deletedByAgentId: + actor.actorType === "agent" ? (actor.agentId ?? null) : null, + deletedByUserId: + actor.actorType === "user" ? (actor.userId ?? null) : null, + deletedByRunId: actor.runId ?? null, + updatedAt: now, + }) + .where( + and( + eq(issueComments.id, commentId), + isNull(issueComments.deletedAt), + ), + ) + .returning(); + + if (!comment) return null; + + await tx + .update(issues) + .set({ updatedAt: now }) + .where(eq(issues.id, comment.issueId)); + + const redacted = redactIssueComment( + comment, + currentUserRedactionOptions.enabled, + ); + await options?.afterTombstone?.(redacted, tx); + + return redacted; + }); + }, + + addComment: async function addComment( + issueId: string, + body: string, + actor: { + agentId?: string; + userId?: string; + runId?: string | null; + onBehalfOfUserId?: string | null; + }, + options?: { + authorType?: IssueCommentAuthorType | null; + presentation?: IssueCommentPresentation | null; + metadata?: IssueCommentMetadata | null; + attachmentIds?: string[]; + authorizationReason?: string | null; + /** Server-only final assistant response, never a tool/progress comment. */ + completionReply?: boolean; + sourceTrust?: typeof issueComments.$inferInsert.sourceTrust; + createdAt?: Date | string | null; + clientRequestId?: string; + /** Server-only: authenticated Paperclip messages also belong in the Slack thread. */ + mirrorToSlack?: boolean; + }, + dbOrTx: any = db, + ): Promise { + if (dbOrTx === db) { + const append = () => + measureRunTransaction(db, "response", async (tx) => { + // Serialize run-authored comments on the issue so a provider retry + // cannot publish the same visible result twice. This needs no schema + // change: the issue row is the transaction fence, and the recursive + // call below performs the lookup and insert while holding it. + await measureRunPhase("response.row_lock", () => tx + .select({ id: issues.id }) + .from(issues) + .where(eq(issues.id, issueId)) + .for("update")); + return addComment(issueId, body, actor, options, tx); + }); + return options?.authorizationReason === + CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON + ? retryNativeChatReviewPresentation(append) + : append(); + } + // The query below locks human comments on caller-owned transactions too, + // sharing the fence with both question creation and Slack settlement. + const issueQuery = dbOrTx + .select({ companyId: issues.companyId, conversationAgentId: issues.conversationAgentId }) + .from(issues) + .where(eq(issues.id, issueId)); + // Caller-owned transactions (including chat and review comments) must + // serialize with question creation before inserting the human comment. + const issue = await (actor.userId || (actor.runId && dbOrTx !== db) ? issueQuery.for("update") : issueQuery) + .then((rows: Array<{ companyId: string; conversationAgentId: string | null }>) => rows[0] ?? null); + + if (!issue) throw notFound("Issue not found"); + + if (issue.conversationAgentId && actor.userId && !(await instanceSettingsService(dbOrTx).getExperimental()).enableAgentChat) { + throw unprocessable("Agent Chat is disabled in Experimental settings"); + } + const currentUserRedactionOptions = { + // Keep every read on the caller's transaction connection. Re-entering + // the outer pool here can deadlock when concurrent transactions fill + // the pool while waiting on the same issue or delivery row. + enabled: (await instanceSettings.getGeneral({ db: dbOrTx })) + .censorUsernameInLogs, + }; + const redactedBody = redactCurrentUserText(body, currentUserRedactionOptions); + if (actor.userId && options?.clientRequestId) { + const [existing] = await dbOrTx.select().from(issueComments).where(and(eq(issueComments.issueId, issueId), + eq(issueComments.authorUserId, actor.userId), eq(issueComments.clientRequestId, options.clientRequestId))); + if (existing) { + if (existing.body !== redactedBody) throw conflict("Message request ID was already used for different content"); + return existing; + } + } + if (issue.conversationAgentId && actor.runId) { + const [run] = await dbOrTx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, actor.runId)); + const [current] = await dbOrTx.select().from(issues).where(eq(issues.id, issueId)); + if (run?.status === "cancelled") throw conflict("This conversation turn was cancelled; it cannot post a reply"); + if (run?.contextSnapshot?.conversationSessionGeneration !== current.conversationSessionGeneration) { + throw conflict("Conversation session changed; this reply belongs to an earlier session"); + } + } + if (options?.completionReply && actor.agentId && actor.runId) { + const delivered = await existingChatCompletionReply(dbOrTx, actor.runId, issueId); + if (delivered) return redactIssueComment(delivered, currentUserRedactionOptions.enabled); + } + const authorType = issueCommentAuthorTypeSchema.parse( + options?.authorType ?? + (actor.agentId ? "agent" : actor.userId ? "user" : "system"), + ); + assertIssueCommentAuthorTypeAllowed(actor, authorType); + const presentation = issueCommentPresentationSchema + .nullable() + .parse(options?.presentation ?? null); + const createdAt = options?.createdAt ? new Date(options.createdAt) : null; + const validCreatedAt = + createdAt && !Number.isNaN(createdAt.getTime()) ? createdAt : null; + // Use one statement timestamp for both columns when the caller did not + // supply a valid historical timestamp. This keeps the comment's + // recency fields equal even when the surrounding transaction started + // earlier, while preserving imported timestamps and the normal default + // updatedAt behavior for those historical rows. + const currentInsertTimestamp = validCreatedAt + ? null + : sql`statement_timestamp()`; + // Invalid/stale run ids must not 500 the insert — null out unknowns. + const createdByRun = await resolveCommentCreatedByRun( + dbOrTx, + issue.companyId, + actor.runId, + ); + const createdByRunId = createdByRun?.id ?? null; + if ( + createdByRunId && + options?.authorizationReason === CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON + ) { + const [presentationRun] = await dbOrTx + .select({ resultJson: heartbeatRuns.resultJson }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, createdByRunId), + eq(heartbeatRuns.companyId, issue.companyId), + ), + ) + .limit(1); + if ( + presentationRun?.resultJson?.finalizationReasonCode === + "governed_response_waiting" && + (parseObject(presentationRun.resultJson.nativeResult).summary !== body || + !(await authorizeNativeChatReviewPresentation( + dbOrTx, + { + companyId: issue.companyId, + issueId, + runId: createdByRunId, + resultJson: presentationRun.resultJson, + }, + "nonblocking", + ))) + ) { + throw conflict( + "This chat response is no longer authorized for external presentation", + { code: "chat_review_response_presentation_denied" }, + ); + } + } + if (actor.runId && !createdByRunId) { + logger.warn( + { issueId, companyId: issue.companyId, runId: actor.runId }, + "dropping invalid createdByRunId for issue comment insert", + ); + } + const attachmentIds = [...new Set(options?.attachmentIds ?? [])]; + if (actor.agentId && attachmentIds.length > 0) { + if (!createdByRunId) { + throw unprocessable( + "Agent comment attachments require the current registered run", + ); + } + // Routes commonly supply an existing transaction for attachment + // comments, so the outer addComment wrapper is not always responsible + // for serialization. Lock both stable parents here before selecting or + // binding files. Concurrent helper calls for the same run then observe + // one another's committed selection count instead of both admitting a + // twenty-first attachment. + const [lockedIssue] = await dbOrTx + .select({ id: issues.id }) + .from(issues) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, issue.companyId), + ), + ) + .for("update"); + if (!lockedIssue) throw notFound("Issue not found"); + const [lockedRun] = await dbOrTx + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, createdByRunId), + eq(heartbeatRuns.companyId, issue.companyId), + eq(heartbeatRuns.agentId, actor.agentId), + ), + ) + .for("update"); + if (!lockedRun) { + throw unprocessable( + "Agent comment attachments require the current registered run", + ); + } + } + const onBehalfOfUserId = actor.agentId + ? await resolveCommentResponsibleUserId( + dbOrTx, + issue.companyId, + createdByRunId, + actor.onBehalfOfUserId, + ) + : null; + const metadata = issueCommentMetadataSchema + .nullable() + .parse( + actor.agentId + ? withAgentCommentAuthorizationMetadata( + options?.metadata ?? null, + options?.authorizationReason, + ) + : (options?.metadata ?? null), + ); + let comment: typeof issueComments.$inferSelect | null = null; + if (createdByRunId) { + const existing = await dbOrTx + .select() + .from(issueComments) + .where( + and( + eq(issueComments.companyId, issue.companyId), + eq(issueComments.issueId, issueId), + eq(issueComments.createdByRunId, createdByRunId), + eq(issueComments.authorType, authorType), + actor.agentId + ? eq(issueComments.authorAgentId, actor.agentId) + : isNull(issueComments.authorAgentId), + eq(issueComments.body, redactedBody), + isNull(issueComments.deletedAt), + ), + ) + .orderBy(issueComments.createdAt, issueComments.id) + .limit(1) + .then( + (rows: Array) => rows[0] ?? null, + ); + if (existing) { + // Heartbeat's presentation resolver can intentionally select the + // exact same prose that the agent already wrote while handling an + // external-chat continuation. The first write is kept internal so + // lifecycle/tool chatter cannot escape; the later, narrowly + // authorized presentation must upgrade that one durable comment + // instead of returning early and letting the generic completion + // milestone win the provider reply slot. + const shouldUpgradeExternalAuthorization = + authorType === "agent" && + metadata?.authorizationReason === + CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON && + existing.metadata?.authorizationReason !== + CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON; + const shouldBindAttachments = + (options?.attachmentIds?.length ?? 0) > 0; + const shouldUpgradeAttachmentAuthorization = + shouldBindAttachments && + isExplicitExternalAgentComment(metadata) && + !isExplicitExternalAgentComment(existing.metadata); + if ( + !shouldUpgradeExternalAuthorization && + !shouldUpgradeAttachmentAuthorization && + !shouldBindAttachments + ) { + if (options?.completionReply && actor.agentId && createdByRunId) await acknowledgeChatCompletionReply(dbOrTx, createdByRunId, existing.id); + return redactIssueComment( + existing, + currentUserRedactionOptions.enabled, + ); + } + await notifyStatusCardIssueChange(dbOrTx, issue.companyId); + comment = + shouldUpgradeExternalAuthorization || + shouldUpgradeAttachmentAuthorization + ? await dbOrTx + .update(issueComments) + .set({ + // Preserve any structured provenance the provisional comment + // carried; the presentation pass only elevates its narrowly + // resolved external-publication authorization. + metadata: { + ...(existing.metadata ?? {}), + ...(metadata ?? {}), + }, + updatedAt: new Date(), + }) + .where( + and( + eq(issueComments.id, existing.id), + eq(issueComments.companyId, issue.companyId), + ), + ) + .returning() + .then( + (rows: Array) => + rows[0] ?? null, + ) + : existing; + } + } + if (!comment) { + [comment] = await measureRunPhase("response.write", () => insertIssueComment(dbOrTx, { + companyId: issue.companyId, + issueId, + authorAgentId: actor.agentId ?? null, + authorUserId: actor.userId ?? null, + onBehalfOfUserId, + authorType, + createdByRunId, + body: redactedBody, + clientRequestId: options?.clientRequestId ?? null, + presentation, + metadata, + sourceTrust: options?.sourceTrust ?? null, + createdAt: validCreatedAt ?? currentInsertTimestamp!, + ...(currentInsertTimestamp + ? { updatedAt: currentInsertTimestamp } + : {}), + })); + } + if (!comment) throw new Error("Failed to create issue comment"); + if (options?.completionReply && actor.agentId && createdByRunId) await acknowledgeChatCompletionReply(dbOrTx, createdByRunId, comment.id); + + const boundAttachments: Array<{ + id: string; + commentId: string; + originalFilename: string | null; + }> = []; + if (attachmentIds.length > 0) { + const attachmentRows = await dbOrTx + .select({ + id: issueAttachments.id, + issueCommentId: issueAttachments.issueCommentId, + originatingRunId: issueAttachments.originatingRunId, + createdByAgentId: assets.createdByAgentId, + originalFilename: assets.originalFilename, + }) + .from(issueAttachments) + .innerJoin(assets, eq(issueAttachments.assetId, assets.id)) + .where( + and( + eq(issueAttachments.companyId, issue.companyId), + eq(issueAttachments.issueId, issueId), + inArray(issueAttachments.id, attachmentIds), + ), + ) + .for("update"); + type CommentAttachmentRow = { + id: string; + issueCommentId: string | null; + originatingRunId: string | null; + createdByAgentId: string | null; + originalFilename: string | null; + }; + const attachmentById = new Map( + attachmentRows.map( + ( + attachment: CommentAttachmentRow, + ): [string, CommentAttachmentRow] => [attachment.id, attachment], + ), + ); + if (attachmentById.size !== attachmentIds.length) { + throw unprocessable( + "Comment attachments must belong to the same task and company", + ); + } + if ( + attachmentRows.some( + (attachment: { issueCommentId: string | null }) => + attachment.issueCommentId !== null && + attachment.issueCommentId !== comment.id, + ) + ) { + throw conflict( + "Comment attachments are already bound to another comment", + { + code: "issue_comment_attachments_already_bound", + companyId: issue.companyId, + issueId, + attachmentIds: attachmentRows + .filter( + (attachment: CommentAttachmentRow) => + attachment.issueCommentId !== null && + attachment.issueCommentId !== comment.id, + ) + .map((attachment: CommentAttachmentRow) => attachment.id), + }, + ); + } + if (actor.agentId && createdByRunId) { + if ( + attachmentRows.some( + (attachment: CommentAttachmentRow) => + attachment.createdByAgentId !== actor.agentId || + attachment.originatingRunId !== createdByRunId, + ) + ) { + throw unprocessable( + "Agent comment attachments must originate from the same agent and current run", + { code: "issue_attachment_run_origin_mismatch" }, + ); + } + const chatBindings = await resolveChatOriginPublicationBindings( + dbOrTx, + issue.companyId, + issueId, + createdByRunId, + ); + if (chatBindings.length > 0) { + const alreadySelected = + await listSelectedChatPresentationAttachments(dbOrTx, { + companyId: issue.companyId, + issueId, + agentId: actor.agentId, + runId: createdByRunId, + }); + const selectedIds = new Set( + alreadySelected.map((attachment) => attachment.id), + ); + for (const attachmentId of attachmentIds) { + selectedIds.add(attachmentId); + } + if (selectedIds.size > MAX_CHAT_PRESENTATION_ATTACHMENTS) { + throw unprocessable( + `An agent run can select at most ${MAX_CHAT_PRESENTATION_ATTACHMENTS} attachments for one chat response`, + { + code: "chat_attachment_selection_limit_exceeded", + limit: MAX_CHAT_PRESENTATION_ATTACHMENTS, + selectedCount: selectedIds.size, + }, + ); + } + } + } + const unboundAttachmentIds = attachmentRows.flatMap( + (attachment: CommentAttachmentRow) => + attachment.issueCommentId === null ? [attachment.id] : [], + ); + const attached = unboundAttachmentIds.length + ? await dbOrTx + .update(issueAttachments) + .set({ issueCommentId: comment.id, updatedAt: new Date() }) + .where( + and( + eq(issueAttachments.companyId, issue.companyId), + eq(issueAttachments.issueId, issueId), + inArray(issueAttachments.id, unboundAttachmentIds), + isNull(issueAttachments.issueCommentId), + ), + ) + .returning({ id: issueAttachments.id }) + : []; + if (attached.length !== unboundAttachmentIds.length) { + throw conflict( + "Comment attachments changed before they could be bound", + ); + } + for (const attachmentId of attachmentIds) { + const attachment = attachmentById.get(attachmentId)!; + boundAttachments.push({ + id: attachment.id, + commentId: comment.id, + originalFilename: attachment.originalFilename, + }); + } + } + + // A chat-origin run binds only the files it explicitly selects to an + // ordinary agent comment while it is still running. That comment stays + // internal until heartbeat chooses the final provider presentation. At + // that point, carry those already-authorized same-agent/same-run files + // forward without inferring any unbound artifact from the run. + if ( + authorType === "agent" && + actor.agentId && + createdByRunId && + metadata?.authorizationReason === + CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON + ) { + const priorRows = await listSelectedChatPresentationAttachments( + dbOrTx, + { + companyId: issue.companyId, + issueId, + agentId: actor.agentId, + runId: createdByRunId, + }, + ); + const selectedIds = new Set( + boundAttachments.map((attachment) => attachment.id), + ); + for (const attachment of priorRows) { + if ( + selectedIds.has(attachment.id) || + !isExplicitExternalAgentComment(attachment.commentMetadata) + ) { + continue; + } + selectedIds.add(attachment.id); + boundAttachments.push({ + id: attachment.id, + commentId: attachment.commentId, + originalFilename: attachment.originalFilename, + }); + } + if (boundAttachments.length > MAX_CHAT_PRESENTATION_ATTACHMENTS) { + throw unprocessable( + `An agent run can select at most ${MAX_CHAT_PRESENTATION_ATTACHMENTS} attachments for one chat response`, + { + code: "chat_attachment_selection_limit_exceeded", + limit: MAX_CHAT_PRESENTATION_ATTACHMENTS, + selectedCount: boundAttachments.length, + }, + ); + } + } + + if (issue.conversationAgentId && actor.userId) { + await dbOrTx.update(issues).set({ conversationState: "active" }).where(eq(issues.id, issueId)); + } + if (options?.mirrorToSlack && actor.userId && authorType === "user") { + await mirrorSlackBoardComment(dbOrTx, comment, { attachmentIds: options.attachmentIds }); + } + if (authorType === "user" || actor.userId) { + await resumeSlackConversation(dbOrTx, issue.companyId, issueId); + } + // Update issue's updatedAt so comment activity is reflected in recency sorting + await dbOrTx + .update(issues) + .set({ updatedAt: new Date() }) + .where(eq(issues.id, issueId)); + + // Only an explicitly authored comment from the run causally woken by an + // inbound chat message is automatically publishable. Presentation, + // recovery, automation, and ordinary internal agent comments stay in + // Paperclip even while a bound conversation is active. + if (authorType === "agent" && isExplicitExternalAgentComment(metadata)) { + // An external-chat run may perform ordinary Paperclip lifecycle or + // bookkeeping writes before its adapter result is finalized. Those + // writes remain internal: only heartbeat's selected final presentation + // may consume this provider response slot. Explicit board "Send to + // channel" publications use the separate publication path. + const chatFinalOwnsProviderReply = + createdByRun !== null && + isExternalChatPresentationContext( + createdByRun.contextSnapshot, + readStringFromRecord(createdByRun.contextSnapshot, "source") === "tool_action_review" && + (await resolveChatOriginPublicationBindings( + dbOrTx, issue.companyId, issueId, createdByRunId, + )).length > 0, + ) && + metadata?.authorizationReason !== + CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON; + const interactionOwnsProviderReply = createdByRunId + ? await hasChatRunOwnedProviderInteraction(dbOrTx, { + companyId: issue.companyId, + issueId, + runId: createdByRunId, + }) + : false; + const bindings = + interactionOwnsProviderReply || chatFinalOwnsProviderReply + ? [] + : await resolveChatOriginPublicationBindings( + dbOrTx, + issue.companyId, + issueId, + createdByRunId, + ); + const publicationCreatedAt = new Date(); + for (const binding of bindings) { + await measureRunPhase("response.publish", () => dbOrTx.transaction(async (tx: DbTransaction) => { + await notifyChatPublicationWork(tx); + return tx + .insert(chatPublications) + .values({ + companyId: binding.companyId, + endpointId: binding.endpointId, + conversationId: binding.conversationId, + issueId, + commentId: comment.id, + idempotencyKey: `comment:${comment.id}:${binding.endpointId}`, + payload: projectSafeChatPublication({ + classification: "external", + source: "agent_comment", + text: redactedBody, + }), + state: "pending", + createdAt: publicationCreatedAt, + updatedAt: publicationCreatedAt, + }) + .onConflictDoNothing(); + })); + for (const [index, attachment] of boundAttachments.entries()) { + const attachmentCreatedAt = new Date( + publicationCreatedAt.getTime() + index + 1, + ); + await measureRunPhase("response.publish", () => dbOrTx.transaction(async (tx: DbTransaction) => { + await notifyChatPublicationWork(tx); + return tx + .insert(chatPublications) + .values({ + companyId: binding.companyId, + endpointId: binding.endpointId, + conversationId: binding.conversationId, + issueId, + commentId: attachment.commentId, + idempotencyKey: `attachment:${attachment.id}:${binding.endpointId}`, + payload: projectSafeChatPublication({ + classification: "external", + source: "agent_comment", + text: `Shared ${attachment.originalFilename ?? "a file"}.`, + attachmentIds: [attachment.id], + }), + state: "pending", + createdAt: attachmentCreatedAt, + updatedAt: attachmentCreatedAt, + }) + .onConflictDoNothing(); + })); + } + } + } + + if ( + authorType === "user" && + actor.userId && + actor.userId !== "board-concierge" && + !createdByRunId + ) { + const { issueThreadInteractionService } = + await import("./issue-thread-interactions.js"); + const expiredInteractions = await issueThreadInteractionService( + dbOrTx, + ).expireRequestConfirmationsSupersededByComment( + { id: issueId, companyId: issue.companyId }, + comment, + { agentId: actor.agentId, userId: actor.userId }, + ); + for (const interaction of expiredInteractions) { + await logActivity(dbOrTx, { + companyId: issue.companyId, + actorType: "user", + actorId: actor.userId, + agentId: actor.agentId ?? null, + runId: createdByRunId, + action: "issue.thread_interaction_expired", + entityType: "issue", + entityId: issueId, + details: { + interactionId: interaction.id, + interactionKind: interaction.kind, + interactionStatus: interaction.status, + source: "issue.comment.service", + result: interaction.result ?? null, + }, + }); + } + } + + return redactIssueComment(comment, currentUserRedactionOptions.enabled); + }, + + createAttachment: async (input: { + issueId: string; + issueCommentId?: string | null; + provider: string; + objectKey: string; + contentType: string; + byteSize: number; + sha256: string; + originalFilename?: string | null; + createdByAgentId?: string | null; + createdByUserId?: string | null; + createdByRunId?: string | null; + }) => { + const issue = await db + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, input.issueId)) + .then((rows) => rows[0] ?? null); + if (!issue) throw notFound("Issue not found"); + + return db.transaction(async (tx) => { + if (input.createdByAgentId && input.issueCommentId) { + const [lockedIssue] = await tx + .select({ id: issues.id }) + .from(issues) + .where( + and( + eq(issues.id, issue.id), + eq(issues.companyId, issue.companyId), + ), + ) + .for("update"); + if (!lockedIssue) throw notFound("Issue not found"); + } + const registeredRun = + input.createdByRunId && isUuidLike(input.createdByRunId) + ? await tx + .select({ + id: heartbeatRuns.id, + status: heartbeatRuns.status, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, input.createdByRunId), + eq(heartbeatRuns.companyId, issue.companyId), + ...(input.createdByAgentId + ? [eq(heartbeatRuns.agentId, input.createdByAgentId)] + : []), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null) + : null; + const registeredRunId = registeredRun?.id ?? null; + const parentComment = input.issueCommentId + ? await tx + .select({ + id: issueComments.id, + companyId: issueComments.companyId, + issueId: issueComments.issueId, + authorType: issueComments.authorType, + authorAgentId: issueComments.authorAgentId, + createdByRunId: issueComments.createdByRunId, + metadata: issueComments.metadata, + }) + .from(issueComments) + .where( + and( + eq(issueComments.id, input.issueCommentId), + isNull(issueComments.deletedAt), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null) + : null; + if (input.issueCommentId && !parentComment) { + throw notFound("Issue comment not found"); + } + if ( + parentComment && + (parentComment.companyId !== issue.companyId || + parentComment.issueId !== issue.id) + ) { + throw unprocessable( + "Attachment comment must belong to same issue and company", + ); + } + if (input.createdByAgentId && parentComment) { + if ( + !registeredRunId || + parentComment.authorType !== "agent" || + parentComment.authorAgentId !== input.createdByAgentId || + parentComment.createdByRunId !== registeredRunId + ) { + throw unprocessable( + "Agent attachment comments must belong to the same agent and current run", + { code: "issue_attachment_parent_run_mismatch" }, + ); + } + const chatBindings = isExplicitExternalAgentComment( + parentComment.metadata, + ) + ? await resolveChatOriginPublicationBindings( + tx, + issue.companyId, + issue.id, + registeredRunId, + ) + : []; + if (chatBindings.length > 0) { + const alreadySelected = + await listSelectedChatPresentationAttachments(tx, { + companyId: issue.companyId, + issueId: issue.id, + agentId: input.createdByAgentId, + runId: registeredRunId, + }); + if (alreadySelected.length >= MAX_CHAT_PRESENTATION_ATTACHMENTS) { + throw unprocessable( + `An agent run can select at most ${MAX_CHAT_PRESENTATION_ATTACHMENTS} attachments for one chat response`, + { + code: "chat_attachment_selection_limit_exceeded", + limit: MAX_CHAT_PRESENTATION_ATTACHMENTS, + selectedCount: alreadySelected.length + 1, + }, + ); + } + } + } + await notifyNativeStatusReassessment(tx, { companyId: issue.companyId, issueId: issue.id }); + const [asset] = await tx + .insert(assets) + .values({ + companyId: issue.companyId, + provider: input.provider, + objectKey: input.objectKey, + contentType: input.contentType, + byteSize: input.byteSize, + sha256: input.sha256, + originalFilename: input.originalFilename ?? null, + createdByAgentId: input.createdByAgentId ?? null, + createdByUserId: input.createdByUserId ?? null, + }) + .returning(); + + const [attachment] = await tx + .insert(issueAttachments) + .values({ + companyId: issue.companyId, + issueId: issue.id, + assetId: asset.id, + issueCommentId: input.issueCommentId ?? null, + originatingRunId: registeredRunId, + }) + .returning(); + const contentPath = `/api/attachments/${attachment.id}/content`; + const [artifactWorkProduct] = registeredRunId + ? await tx + .insert(issueWorkProducts) + .values({ + companyId: issue.companyId, + issueId: issue.id, + type: "artifact", + provider: "paperclip", + externalId: attachment.id, + title: asset.originalFilename ?? "Attachment", + status: "active", + reviewState: "none", + isPrimary: false, + healthStatus: "unknown", + metadata: { + attachmentId: attachment.id, + contentType: asset.contentType, + byteSize: asset.byteSize, + contentPath, + openPath: contentPath, + downloadPath: `${contentPath}?download=1`, + originalFilename: asset.originalFilename, + }, + createdByRunId: registeredRunId, + }) + .returning({ id: issueWorkProducts.id }) + : []; + + if ( + input.createdByAgentId && + registeredRunId && + parentComment?.authorType === "agent" && + parentComment.authorAgentId === input.createdByAgentId && + parentComment.createdByRunId === registeredRunId && + parentComment.metadata?.authorizationReason === + CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON + ) { + const bindings = await resolveChatOriginPublicationBindings( + tx, + issue.companyId, + issue.id, + registeredRunId, + ); + for (const binding of bindings) { + await notifyChatPublicationWork(tx); + await tx + .insert(chatPublications) + .values({ + companyId: issue.companyId, + endpointId: binding.endpointId, + conversationId: binding.conversationId, + issueId: issue.id, + commentId: parentComment.id, + idempotencyKey: `attachment:${attachment.id}:${binding.endpointId}`, + payload: projectSafeChatPublication({ + classification: "external", + source: "agent_comment", + text: `Shared ${asset.originalFilename ?? "a file"}.`, + attachmentIds: [attachment.id], + }), + state: "pending", + }) + .onConflictDoNothing(); + } + } + + return { + id: attachment.id, + companyId: attachment.companyId, + issueId: attachment.issueId, + issueCommentId: attachment.issueCommentId, + originatingRunId: attachment.originatingRunId, + assetId: attachment.assetId, + provider: asset.provider, + objectKey: asset.objectKey, + contentType: asset.contentType, + byteSize: asset.byteSize, + sha256: asset.sha256, + originalFilename: asset.originalFilename, + createdByAgentId: asset.createdByAgentId, + createdByUserId: asset.createdByUserId, + createdAt: attachment.createdAt, + updatedAt: attachment.updatedAt, + artifactWorkProductId: artifactWorkProduct?.id ?? null, + }; + }); + }, + + listAttachments: async (issueId: string) => + db + .select({ + id: issueAttachments.id, + companyId: issueAttachments.companyId, + issueId: issueAttachments.issueId, + issueCommentId: issueAttachments.issueCommentId, + originatingRunId: issueAttachments.originatingRunId, + assetId: issueAttachments.assetId, + provider: assets.provider, + objectKey: assets.objectKey, + contentType: assets.contentType, + byteSize: assets.byteSize, + sha256: assets.sha256, + originalFilename: assets.originalFilename, + createdByAgentId: assets.createdByAgentId, + createdByUserId: assets.createdByUserId, + createdAt: issueAttachments.createdAt, + updatedAt: issueAttachments.updatedAt, + }) + .from(issueAttachments) + .innerJoin(assets, eq(issueAttachments.assetId, assets.id)) + .where(eq(issueAttachments.issueId, issueId)) + .orderBy(desc(issueAttachments.createdAt)), + + getAttachmentById: async (id: string) => + db + .select({ + id: issueAttachments.id, + companyId: issueAttachments.companyId, + issueId: issueAttachments.issueId, + issueCommentId: issueAttachments.issueCommentId, + originatingRunId: issueAttachments.originatingRunId, + assetId: issueAttachments.assetId, + provider: assets.provider, + objectKey: assets.objectKey, + contentType: assets.contentType, + byteSize: assets.byteSize, + sha256: assets.sha256, + originalFilename: assets.originalFilename, + createdByAgentId: assets.createdByAgentId, + createdByUserId: assets.createdByUserId, + createdAt: issueAttachments.createdAt, + updatedAt: issueAttachments.updatedAt, + }) + .from(issueAttachments) + .innerJoin(assets, eq(issueAttachments.assetId, assets.id)) + .where(eq(issueAttachments.id, id)) + .then((rows) => rows[0] ?? null), + + removeAttachment: async (id: string) => + db.transaction(async (tx) => { + const existing = await tx + .select({ + id: issueAttachments.id, + companyId: issueAttachments.companyId, + issueId: issueAttachments.issueId, + issueCommentId: issueAttachments.issueCommentId, + originatingRunId: issueAttachments.originatingRunId, + assetId: issueAttachments.assetId, + provider: assets.provider, + objectKey: assets.objectKey, + contentType: assets.contentType, + byteSize: assets.byteSize, + sha256: assets.sha256, + originalFilename: assets.originalFilename, + createdByAgentId: assets.createdByAgentId, + createdByUserId: assets.createdByUserId, + createdAt: issueAttachments.createdAt, + updatedAt: issueAttachments.updatedAt, + }) + .from(issueAttachments) + .innerJoin(assets, eq(issueAttachments.assetId, assets.id)) + .where(eq(issueAttachments.id, id)) + .then((rows) => rows[0] ?? null); + if (!existing) return null; + + await tx.delete(issueAttachments).where(eq(issueAttachments.id, id)); + await tx.delete(assets).where(eq(assets.id, existing.assetId)); + return existing; + }), + + findMentionedAgents: async (companyId: string, body: string) => { + const explicitAgentMentionIds = extractAgentMentionIds(body); + if (explicitAgentMentionIds.length === 0) return []; + + const rows = await db + .select({ id: agents.id }) + .from(agents) + .where(eq(agents.companyId, companyId)); + const companyAgentIds = new Set(rows.map((agent) => agent.id)); + return explicitAgentMentionIds.filter((agentId) => + companyAgentIds.has(agentId), + ); + }, + + findMentionedProjectIds: async ( + issueId: string, + opts?: { includeCommentBodies?: boolean }, + ) => { + const issue = await db + .select({ + companyId: issues.companyId, + title: issues.title, + description: issues.description, + }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + if (!issue) return []; + + const mentionedIds = new Set(); + for (const source of [issue.title, issue.description ?? ""]) { + for (const projectId of extractProjectMentionIds(source)) { + mentionedIds.add(projectId); + } + } + + if (opts?.includeCommentBodies !== false) { + const comments = await db + .select({ body: issueComments.body }) + .from(issueComments) + .where( + and( + eq(issueComments.issueId, issueId), + isNull(issueComments.deletedAt), + ), + ); + + for (const comment of comments) { + for (const projectId of extractProjectMentionIds(comment.body)) { + mentionedIds.add(projectId); + } + } + } + + if (mentionedIds.size === 0) return []; + + const rows = await db + .select({ id: projects.id }) + .from(projects) + .where( + and( + eq(projects.companyId, issue.companyId), + inArray(projects.id, [...mentionedIds]), + ), + ); + const valid = new Set(rows.map((row) => row.id)); + return [...mentionedIds].filter((projectId) => valid.has(projectId)); + }, + + getAncestors: async (issueId: string) => { + const raw: Array<{ + id: string; + identifier: string | null; + title: string; + description: string | null; + status: string; + priority: string; + assigneeAgentId: string | null; + projectId: string | null; + goalId: string | null; + }> = []; + const visited = new Set([issueId]); + const start = await db + .select() + .from(issues) + .where(eq(issues.id, issueId)) + .then((r) => r[0] ?? null); + let currentId = start?.parentId ?? null; + while (currentId && !visited.has(currentId) && raw.length < 50) { + visited.add(currentId); + const parent = await db + .select({ + id: issues.id, + identifier: issues.identifier, + title: issues.title, + description: issues.description, + status: issues.status, + priority: issues.priority, + assigneeAgentId: issues.assigneeAgentId, + projectId: issues.projectId, + goalId: issues.goalId, + parentId: issues.parentId, + }) + .from(issues) + .where(eq(issues.id, currentId)) + .then((r) => r[0] ?? null); + if (!parent) break; + raw.push({ + id: parent.id, + identifier: parent.identifier ?? null, + title: parent.title, + description: parent.description ?? null, + status: parent.status, + priority: parent.priority, + assigneeAgentId: parent.assigneeAgentId ?? null, + projectId: parent.projectId ?? null, + goalId: parent.goalId ?? null, + }); + currentId = parent.parentId ?? null; + } + + // Batch-fetch referenced projects and goals + const projectIds = [ + ...new Set( + raw.map((a) => a.projectId).filter((id): id is string => id != null), + ), + ]; + const goalIds = [ + ...new Set( + raw.map((a) => a.goalId).filter((id): id is string => id != null), + ), + ]; + + const projectMap = new Map< + string, + { + id: string; + name: string; + description: string | null; + status: string; + goalId: string | null; + workspaces: Array<{ + id: string; + companyId: string; + projectId: string; + name: string; + cwd: string | null; + repoUrl: string | null; + repoRef: string | null; + metadata: Record | null; + isPrimary: boolean; + createdAt: Date; + updatedAt: Date; + }>; + primaryWorkspace: { + id: string; + companyId: string; + projectId: string; + name: string; + cwd: string | null; + repoUrl: string | null; + repoRef: string | null; + metadata: Record | null; + isPrimary: boolean; + createdAt: Date; + updatedAt: Date; + } | null; + } + >(); + const goalMap = new Map< + string, + { + id: string; + title: string; + description: string | null; + level: string; + status: string; + } + >(); + + if (projectIds.length > 0) { + const workspaceRows = await db + .select() + .from(projectWorkspaces) + .where(inArray(projectWorkspaces.projectId, projectIds)) + .orderBy( + desc(projectWorkspaces.isPrimary), + asc(projectWorkspaces.createdAt), + asc(projectWorkspaces.id), + ); + const workspaceMap = new Map< + string, + Array<(typeof workspaceRows)[number]> + >(); + for (const workspace of workspaceRows) { + const existing = workspaceMap.get(workspace.projectId); + if (existing) existing.push(workspace); + else workspaceMap.set(workspace.projectId, [workspace]); + } + + const rows = await db + .select({ + id: projects.id, + name: projects.name, + description: projects.description, + status: projects.status, + goalId: projects.goalId, + }) + .from(projects) + .where(inArray(projects.id, projectIds)); + for (const r of rows) { + const projectWorkspaceRows = workspaceMap.get(r.id) ?? []; + const workspaces = projectWorkspaceRows.map((workspace) => ({ + id: workspace.id, + companyId: workspace.companyId, + projectId: workspace.projectId, + name: workspace.name, + cwd: workspace.cwd, + repoUrl: workspace.repoUrl ?? null, + repoRef: workspace.repoRef ?? null, + metadata: + (workspace.metadata as Record | null) ?? null, + isPrimary: workspace.isPrimary, + createdAt: workspace.createdAt, + updatedAt: workspace.updatedAt, + })); + const primaryWorkspace = + workspaces.find((workspace) => workspace.isPrimary) ?? + workspaces[0] ?? + null; + projectMap.set(r.id, { + ...r, + workspaces, + primaryWorkspace, + }); + // Also collect goalIds from projects + if (r.goalId && !goalIds.includes(r.goalId)) goalIds.push(r.goalId); + } + } + + if (goalIds.length > 0) { + const rows = await db + .select({ + id: goals.id, + title: goals.title, + description: goals.description, + level: goals.level, + status: goals.status, + }) + .from(goals) + .where(inArray(goals.id, goalIds)); + for (const r of rows) goalMap.set(r.id, r); + } + + return raw.map((a) => ({ + ...a, + project: a.projectId ? (projectMap.get(a.projectId) ?? null) : null, + goal: a.goalId ? (goalMap.get(a.goalId) ?? null) : null, + })); + }, + }; + + type IssueServiceApi = typeof service & { + updateForCompany: ( + id: string, + companyId: string, + data: Parameters[1], + dbOrTx?: any, + postCommitActivityPublications?: ActivityPublication[], + postCommitActions?: IssuePostCommitAction[], + ) => ReturnType; + }; + const serviceApi = service as IssueServiceApi; + + // A company-scoped wrapper around `update`. It passes the company as a + // guard on every read, lock, and write predicate, so a caller with only + // a company id and an issue id cannot update an issue in another company. + serviceApi.updateForCompany = async ( + id, + companyId, + data, + dbOrTx = db, + postCommitActivityPublications, + postCommitActions, + ) => { + return service.update( + id, + { ...data, companyGuard: companyId }, + dbOrTx, + postCommitActivityPublications, + postCommitActions, + ); + }; + + return serviceApi; +}