import { notifyInstructionCollection } from "../../../services/instruction-collection-notifications.js"; import { and, asc, desc, eq, gt, gte, inArray, ne, notInArray, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { agents, agentWakeupRequests, activityLog, heartbeatRunWatchdogDecisions, heartbeatRuns, environmentLeases, issues, workspaceOperations, } from "@paperclipai/db"; import { parseObject } from "../../../adapters/utils.js"; import { visibleIssueCondition } from "../../../services/issue-visibility.js"; import { logActivity } from "../../../services/activity-log.js"; import { appendHeartbeatRunEvent } from "../../../services/heartbeat-run-events.js"; import { emitAgentTaskRun } from "../../../services/agent-task-run-telemetry.js"; import { executeIssuePostCommitActions, issueService, type IssuePostCommitAction, } from "../../../services/issues.js"; import { issueRecoveryActionService } from "../../../services/issue-recovery-actions.js"; import { RECOVERY_ORIGIN_KINDS } from "../../../services/recovery/origins.js"; import { isTerminalIssueStatus } from "../domain/policy.js"; import type { WatchdogRunReader, WatchdogWriter } from "../application/ports.js"; import type { EvaluationIssueSnapshot, FoldOutcome, FoldSourceResolvedRunInput, RecordDecisionInput, RunSnapshot, SourceIssueSnapshot, WatchdogDecisionRecord, } from "../application/types.js"; const STALE_ACTIVE_RUN_EVALUATION_ORIGIN_KIND = RECOVERY_ORIGIN_KINDS.staleActiveRunEvaluation; const RECOVERY_ORIGIN_KIND_VALUES = new Set(Object.values(RECOVERY_ORIGIN_KINDS)); function isRecoveryOriginKind(originKind: string | null): boolean { return originKind !== null && RECOVERY_ORIGIN_KIND_VALUES.has(originKind); } function issueContextId(contextSnapshot: unknown): string | null { const context = parseObject(contextSnapshot); const issueId = context.issueId ?? context.taskId; return typeof issueId === "string" && issueId.length > 0 ? issueId : null; } function toRunSnapshot(row: typeof heartbeatRuns.$inferSelect): RunSnapshot { return { id: row.id, companyId: row.companyId, agentId: row.agentId, status: row.status, lastOutputAt: row.lastOutputAt, lastOutputSeq: row.lastOutputSeq, lastOutputStream: row.lastOutputStream, processStartedAt: row.processStartedAt, startedAt: row.startedAt, createdAt: row.createdAt, sourceIssueId: issueContextId(row.contextSnapshot), resultJson: row.resultJson, wakeupRequestId: row.wakeupRequestId, processPid: row.processPid, processGroupId: row.processGroupId, }; } function runtimeSettlementPending(companyId: string, runId: string) { return sql`( ${heartbeatRuns.runtimeMode} = 'native' or coalesce(${heartbeatRuns.resultJson}->>'workspaceRestoreFailure', '') <> '' or exists ( select 1 from ${environmentLeases} where ${environmentLeases.companyId} = ${companyId} and ${environmentLeases.heartbeatRunId} = ${runId} and ${environmentLeases.providerLeaseId} is not null and ${environmentLeases.status} in ('active', 'acquiring') ) or exists ( select 1 from ${workspaceOperations} where ${workspaceOperations.companyId} = ${companyId} and ${workspaceOperations.heartbeatRunId} = ${runId} and ${workspaceOperations.phase} = 'workspace_finalize' and ${workspaceOperations.status} in ('running', 'failed') ) )`; } export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & WatchdogWriter { const issuesSvc = issueService(db); const recoveryActionsSvc = issueRecoveryActionService(db); async function findCandidateSilentRuns(input: { companyId?: string; suspicionBefore: Date; issueCreatedAtGte?: Date | null; }): Promise { const rows = await db .select() .from(heartbeatRuns) .where( and( input.companyId ? eq(heartbeatRuns.companyId, input.companyId) : undefined, eq(heartbeatRuns.status, "running"), sql`coalesce(${heartbeatRuns.lastOutputAt}, ${heartbeatRuns.processStartedAt}, ${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt}) <= ${input.suspicionBefore.toISOString()}::timestamptz`, ), ) .orderBy(asc(heartbeatRuns.createdAt)) .limit(100); let candidates = rows.map(toRunSnapshot); if (input.issueCreatedAtGte) { const issueCreatedAtGte = input.issueCreatedAtGte; const issueIds = [...new Set(candidates.flatMap((run) => { return run.sourceIssueId ? [run.sourceIssueId] : []; }))]; const eligibleIssueIds = new Set( issueIds.length > 0 ? (await db.select({ id: issues.id }).from(issues).where(and( inArray(issues.id, issueIds), gte(issues.createdAt, issueCreatedAtGte), ))).map((issue) => issue.id) : [], ); candidates = candidates.filter((run) => { return run.sourceIssueId !== null && eligibleIssueIds.has(run.sourceIssueId); }); } return candidates; } async function findRunForCompany(companyId: string, runId: string): Promise { const [row] = await db .select() .from(heartbeatRuns) .where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId))) .limit(1); return row ? toRunSnapshot(row) : null; } async function isRuntimeSettlementPending(companyId: string, runId: string): Promise { const [row] = await db.select({ pending: sql`${heartbeatRuns.status} <> 'running' or ${runtimeSettlementPending(companyId, runId)}`, }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId))); return row?.pending ?? true; } async function findLatestDecision(companyId: string, runId: string, now: Date) { const [quietUntilRows, dismissedRows] = await Promise.all([ db .select({ decision: heartbeatRunWatchdogDecisions.decision, snoozedUntil: heartbeatRunWatchdogDecisions.snoozedUntil, }) .from(heartbeatRunWatchdogDecisions) .where( and( eq(heartbeatRunWatchdogDecisions.companyId, companyId), eq(heartbeatRunWatchdogDecisions.runId, runId), inArray(heartbeatRunWatchdogDecisions.decision, ["snooze", "continue"]), gt(heartbeatRunWatchdogDecisions.snoozedUntil, now), ), ) .orderBy(desc(heartbeatRunWatchdogDecisions.createdAt)) .limit(1), db .select({ id: heartbeatRunWatchdogDecisions.id }) .from(heartbeatRunWatchdogDecisions) .where( and( eq(heartbeatRunWatchdogDecisions.companyId, companyId), eq(heartbeatRunWatchdogDecisions.runId, runId), eq(heartbeatRunWatchdogDecisions.decision, "dismissed_false_positive"), ), ) .limit(1), ]); const quietUntilRow = quietUntilRows[0]; return { dismissedFalsePositive: dismissedRows.length > 0, quietUntilDecision: quietUntilRow && quietUntilRow.snoozedUntil ? { decision: quietUntilRow.decision as "snooze" | "continue", snoozedUntil: quietUntilRow.snoozedUntil } : null, }; } function selectEvaluationIssueSnapshot() { return { id: issues.id, identifier: issues.identifier, status: issues.status, assigneeAgentId: issues.assigneeAgentId, companyId: issues.companyId, originKind: issues.originKind, originId: issues.originId, hiddenAt: issues.hiddenAt, }; } async function findOpenStaleRunEvaluation(companyId: string, runId: string): Promise { const [row] = await db .select(selectEvaluationIssueSnapshot()) .from(issues) .where( and( eq(issues.companyId, companyId), eq(issues.originKind, STALE_ACTIVE_RUN_EVALUATION_ORIGIN_KIND), eq(issues.originId, runId), visibleIssueCondition(), notInArray(issues.status, ["done", "cancelled"]), ), ) .limit(1); return row ?? null; } async function findEvaluationIssueById(companyId: string, issueId: string): Promise { const [row] = await db .select(selectEvaluationIssueSnapshot()) .from(issues) .where(and(eq(issues.id, issueId), eq(issues.companyId, companyId))) .limit(1); return row ?? null; } async function findLatestSameRunTerminalEvidence(companyId: string, input: { runId: string; sourceIssueId: string; sourceIssueStatus: string; evidenceAfter: Date | null; }) { if (!isTerminalIssueStatus(input.sourceIssueStatus)) return null; const activityPredicates = [ eq(activityLog.companyId, companyId), eq(activityLog.runId, input.runId), eq(activityLog.action, "issue.updated"), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, input.sourceIssueId), sql`${activityLog.details} ->> 'status' = ${input.sourceIssueStatus}`, ]; if (input.evidenceAfter) { activityPredicates.push(gte(activityLog.createdAt, input.evidenceAfter)); } const activity = await db .select({ id: activityLog.id, createdAt: activityLog.createdAt, action: activityLog.action }) .from(activityLog) .where(and(...activityPredicates)) .orderBy(desc(activityLog.createdAt)) .limit(1) .then((rows) => rows[0] ?? null); return activity ? { kind: "activity" as const, id: activity.id, createdAt: activity.createdAt, action: activity.action } : null; } async function findSourceIssue(companyId: string, issueId: string): Promise { const [row] = await db .select({ id: issues.id, identifier: issues.identifier, status: issues.status, originKind: issues.originKind }) .from(issues) .where(and(eq(issues.companyId, companyId), eq(issues.id, issueId), visibleIssueCondition())) .limit(1); return row ? { ...row, isRecoveryOriginKind: isRecoveryOriginKind(row.originKind) } : null; } async function findRunningAgent(companyId: string, agentId: string) { const [row] = await db .select({ id: agents.id, companyId: agents.companyId, adapterType: agents.adapterType }) .from(agents) .where(and(eq(agents.id, agentId), eq(agents.companyId, companyId))) .limit(1); return row ?? null; } async function recordDecision(companyId: string, input: RecordDecisionInput): Promise { const [row] = await db .insert(heartbeatRunWatchdogDecisions) .values({ companyId, runId: input.runId, evaluationIssueId: input.evaluationIssueId, decision: input.decision, snoozedUntil: input.snoozedUntil, reason: input.reason, createdByAgentId: input.createdByAgentId, createdByUserId: input.createdByUserId, createdByRunId: input.createdByRunId, }) .returning(); await logActivity(db, { companyId, actorType: input.actor.type === "agent" ? "agent" : "user", actorId: input.actor.type === "agent" ? input.actor.agentId ?? "agent" : input.actor.type === "board" ? input.actor.userId ?? "board" : "unknown", agentId: input.actor.type === "agent" ? input.actor.agentId ?? null : null, runId: input.runId, action: input.decision === "snooze" ? "heartbeat.watchdog_snoozed" : "heartbeat.watchdog_decision_recorded", entityType: "heartbeat_run", entityId: input.runId, details: { source: "recovery.record_watchdog_decision", decision: input.decision, evaluationIssueId: input.evaluationIssueId, snoozedUntil: input.snoozedUntil?.toISOString() ?? null, reason: input.reason, }, }); return { ...row, decision: row.decision as WatchdogDecisionRecord["decision"] }; } async function foldSourceResolvedRun(companyId: string, input: FoldSourceResolvedRunInput): Promise { const finalRunStatus = input.sourceIssue.status === "cancelled" ? "cancelled" : "succeeded"; const postCommitIssueActions: IssuePostCommitAction[] = []; const resultJson = { ...parseObject(input.run.resultJson), sourceResolvedWatchdogFold: { sourceIssueId: input.sourceIssue.id, sourceIssueIdentifier: input.sourceIssue.identifier, sourceIssueStatus: input.sourceIssue.status, sameRunEvidenceKind: input.evidence.kind, sameRunEvidenceId: input.evidence.id, sameRunEvidenceAt: input.evidence.createdAt.toISOString(), silenceStartedAt: input.silenceStartedAt?.toISOString() ?? null, silenceAgeMs: input.silenceAgeMs, evaluationIssueId: input.existingEvaluation?.id ?? null, evaluationIssueIdentifier: input.existingEvaluation?.identifier ?? null, cleanup: input.cleanup, }, }; const transactionResult = await db.transaction(async (tx) => { await notifyInstructionCollection(tx, { companyId, runId: input.run.id }); const [updatedRun] = await tx .update(heartbeatRuns) .set({ status: finalRunStatus, finishedAt: input.now, error: null, errorCode: null, resultJson, updatedAt: input.now, }) .where(and( eq(heartbeatRuns.id, input.run.id), eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.status, "running"), sql`not ${runtimeSettlementPending(companyId, input.run.id)}`, )) .returning(); if (!updatedRun) return null; if (input.run.wakeupRequestId) { await tx .update(agentWakeupRequests) .set({ status: finalRunStatus === "succeeded" ? "completed" : "cancelled", finishedAt: input.now, error: null, updatedAt: input.now, }) .where(and( eq(agentWakeupRequests.id, input.run.wakeupRequestId), eq(agentWakeupRequests.companyId, companyId), // A late success fold cannot restore cancelled assignment authority. ne(agentWakeupRequests.status, "cancelled"), )); } await tx .update(issues) .set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, updatedAt: input.now, }) .where(and( eq(issues.id, input.sourceIssue.id), eq(issues.companyId, companyId), eq(issues.executionRunId, input.run.id), )); if (input.existingEvaluation && !isTerminalIssueStatus(input.existingEvaluation.status)) { const updatedEvaluation = await issuesSvc.update( input.existingEvaluation.id, { status: "done" }, tx, undefined, postCommitIssueActions, ); if (!updatedEvaluation) { throw new Error("Evaluation issue disappeared during source-resolved watchdog fold"); } await issuesSvc.addComment(input.existingEvaluation.id, [ "Source-resolved watchdog fold.", "", `- Source issue: ${input.sourceIssue.identifier ?? input.sourceIssue.id}`, `- Run: \`${input.run.id}\``, `- Same-run evidence: \`${input.evidence.kind}:${input.evidence.id}\` at ${input.evidence.createdAt.toISOString()}`, "- Outcome: false positive; the source issue already reached a terminal disposition from this run.", ].join("\n"), { runId: input.run.id }, undefined, tx); } const activeRecoveryAction = await recoveryActionsSvc.getActiveForIssue( companyId, input.sourceIssue.id, tx, ); if (activeRecoveryAction?.kind === "active_run_watchdog") { await recoveryActionsSvc.resolveActiveForIssue({ companyId, sourceIssueId: input.sourceIssue.id, actionId: activeRecoveryAction.id, status: "resolved", outcome: "false_positive", resolutionNote: "Source issue reached a terminal disposition through durable same-run activity; watchdog folded as source-resolved.", }, tx); } const [decision] = await tx .insert(heartbeatRunWatchdogDecisions) .values({ companyId, runId: input.run.id, evaluationIssueId: input.existingEvaluation?.id ?? null, decision: "dismissed_false_positive", reason: "Source issue already reached a terminal disposition through durable same-run activity.", createdByRunId: input.run.id, }) .returning(); await appendHeartbeatRunEvent(tx as unknown as Db, { companyId, runId: input.run.id, agentId: input.run.agentId, eventType: "lifecycle", stream: "system", level: input.cleanup.outcome === "failed" ? "warn" : "info", message: "Source-resolved watchdog fold finalized stale active run", payload: resultJson.sourceResolvedWatchdogFold, }); await logActivity(tx as unknown as Db, { companyId, actorType: "system", actorId: "system", agentId: input.run.agentId, runId: input.run.id, action: "heartbeat.output_stale_source_resolved", entityType: "heartbeat_run", entityId: input.run.id, details: { source: "recovery.scan_silent_active_runs", sourceIssueId: input.sourceIssue.id, sourceIssueIdentifier: input.sourceIssue.identifier, sourceIssueStatus: input.sourceIssue.status, evaluationIssueId: input.existingEvaluation?.id ?? null, watchdogDecisionId: decision.id, sameRunEvidenceKind: input.evidence.kind, sameRunEvidenceId: input.evidence.id, sameRunEvidenceAt: input.evidence.createdAt.toISOString(), cleanup: input.cleanup, }, }); const [runningCountRow] = await tx .select({ count: sql`count(*)::int` }) .from(heartbeatRuns) .where(and( eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.agentId, input.run.agentId), eq(heartbeatRuns.status, "running"), )); const runningCount = Number(runningCountRow?.count ?? 0); const nextAgentStatus = runningCount > 0 ? "running" : "idle"; await tx .update(agents) .set({ status: nextAgentStatus, lastHeartbeatAt: new Date(), updatedAt: new Date() }) .where(and( eq(agents.id, input.run.agentId), eq(agents.companyId, companyId), notInArray(agents.status, ["paused", "terminated"]), eq(agents.lifecycleState, "ready"), )); return updatedRun; }); if (!transactionResult) return { kind: "stale" }; const finalizedRun = transactionResult; await executeIssuePostCommitActions(db, postCommitIssueActions); // Telemetry is best-effort background work; it must not delay the // watchdog fold's caller, so fire it and do not await it. void emitAgentTaskRun(db, finalizedRun); return { kind: "folded", evaluationIssueId: input.existingEvaluation?.id ?? null }; } return { findCandidateSilentRuns, findRunForCompany, isRuntimeSettlementPending, findLatestDecision, findOpenStaleRunEvaluation, findEvaluationIssueById, findLatestSameRunTerminalEvidence, findSourceIssue, findRunningAgent, recordDecision, foldSourceResolvedRun, }; }