PaperClipAI/server/src/modules/active-run-watchdog/adapters/postgres.ts

import { DELIVERY_QUEUES, notifyDeliveryWork } from "../../../services/delivery-work-notifications.js";
import { notifyExternalAgentSettlement } from "../../external-agents/index.js";
import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js";
import { notifyCancelledWorkspacePreservation } from "../../../services/cancelled-workspace-preservation-notifications.js";
import { notifyInstructionCollection } from "../../../services/instruction-collection-notifications.js";
import { and, asc, desc, eq, gt, gte, inArray, ne, notInArray, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
  agents,
  agentWakeupRequests,
  activityLog,
  heartbeatRunWatchdogDecisions,
  heartbeatRuns,
  environmentLeases,
  issues,
  workspaceOperations,
} from "@paperclipai/db";
import { parseObject } from "../../../adapters/utils.js";
import { visibleIssueCondition } from "../../../services/issue-visibility.js";
import { logActivity } from "../../../services/activity-log.js";
import { appendHeartbeatRunEvent } from "../../../services/heartbeat-run-events.js";
import { emitAgentTaskRun } from "../../../services/agent-task-run-telemetry.js";
import {
  executeIssuePostCommitActions,
  issueService,
  type IssuePostCommitAction,
} from "../../../services/issues.js";
import { issueRecoveryActionService } from "../../../services/issue-recovery-actions.js";
import { RECOVERY_ORIGIN_KINDS } from "../../../services/recovery/origins.js";
import { isTerminalIssueStatus } from "../domain/policy.js";
import type { WatchdogRunReader, WatchdogWriter } from "../application/ports.js";
import type {
  EvaluationIssueSnapshot,
  FoldOutcome,
  FoldSourceResolvedRunInput,
  RecordDecisionInput,
  RunSnapshot,
  SourceIssueSnapshot,
  WatchdogDecisionRecord,
} from "../application/types.js";

const STALE_ACTIVE_RUN_EVALUATION_ORIGIN_KIND = RECOVERY_ORIGIN_KINDS.staleActiveRunEvaluation;
const RECOVERY_ORIGIN_KIND_VALUES = new Set<string>(Object.values(RECOVERY_ORIGIN_KINDS));

function isRecoveryOriginKind(originKind: string | null): boolean {
  return originKind !== null && RECOVERY_ORIGIN_KIND_VALUES.has(originKind);
}

function issueContextId(contextSnapshot: unknown): string | null {
  const context = parseObject(contextSnapshot);
  const issueId = context.issueId ?? context.taskId;
  return typeof issueId === "string" && issueId.length > 0 ? issueId : null;
}

function toRunSnapshot(row: typeof heartbeatRuns.$inferSelect): RunSnapshot {
  return {
    id: row.id,
    companyId: row.companyId,
    agentId: row.agentId,
    status: row.status,
    lastOutputAt: row.lastOutputAt,
    lastOutputSeq: row.lastOutputSeq,
    lastOutputStream: row.lastOutputStream,
    processStartedAt: row.processStartedAt,
    startedAt: row.startedAt,
    createdAt: row.createdAt,
    sourceIssueId: issueContextId(row.contextSnapshot),
    resultJson: row.resultJson,
    wakeupRequestId: row.wakeupRequestId,
    processPid: row.processPid,
    processGroupId: row.processGroupId,
  };
}

function runtimeSettlementPending(companyId: string, runId: string) {
  return sql<boolean>`(
    ${heartbeatRuns.runtimeMode} = 'native'
    or coalesce(${heartbeatRuns.resultJson}->>'workspaceRestoreFailure', '') <> ''
    or exists (
      select 1 from ${environmentLeases}
      where ${environmentLeases.companyId} = ${companyId}
        and ${environmentLeases.heartbeatRunId} = ${runId}
        and ${environmentLeases.providerLeaseId} is not null
        and ${environmentLeases.status} in ('active', 'acquiring')
    )
    or exists (
      select 1 from ${workspaceOperations}
      where ${workspaceOperations.companyId} = ${companyId}
        and ${workspaceOperations.heartbeatRunId} = ${runId}
        and ${workspaceOperations.phase} = 'workspace_finalize'
        and ${workspaceOperations.status} in ('running', 'failed')
    )
  )`;
}

export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & WatchdogWriter {
  const issuesSvc = issueService(db);
  const recoveryActionsSvc = issueRecoveryActionService(db);

  async function findCandidateSilentRuns(input: {
    companyId?: string;
    suspicionBefore: Date;
    issueCreatedAtGte?: Date | null;
  }): Promise<RunSnapshot[]> {
    const rows = await db
      .select()
      .from(heartbeatRuns)
      .where(
        and(
          input.companyId ? eq(heartbeatRuns.companyId, input.companyId) : undefined,
          eq(heartbeatRuns.status, "running"),
          sql`coalesce(${heartbeatRuns.lastOutputAt}, ${heartbeatRuns.processStartedAt}, ${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt}) <= ${input.suspicionBefore.toISOString()}::timestamptz`,
        ),
      )
      .orderBy(asc(heartbeatRuns.createdAt))
      .limit(100);

    let candidates = rows.map(toRunSnapshot);

    if (input.issueCreatedAtGte) {
      const issueCreatedAtGte = input.issueCreatedAtGte;
      const issueIds = [...new Set(candidates.flatMap((run) => {
        return run.sourceIssueId ? [run.sourceIssueId] : [];
      }))];
      const eligibleIssueIds = new Set(
        issueIds.length > 0
          ? (await db.select({ id: issues.id }).from(issues).where(and(
              inArray(issues.id, issueIds),
              gte(issues.createdAt, issueCreatedAtGte),
            ))).map((issue) => issue.id)
          : [],
      );
      candidates = candidates.filter((run) => {
        return run.sourceIssueId !== null && eligibleIssueIds.has(run.sourceIssueId);
      });
    }

    return candidates;
  }

  async function findRunForCompany(companyId: string, runId: string): Promise<RunSnapshot | null> {
    const [row] = await db
      .select()
      .from(heartbeatRuns)
      .where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId)))
      .limit(1);
    return row ? toRunSnapshot(row) : null;
  }

  async function isRuntimeSettlementPending(companyId: string, runId: string): Promise<boolean> {
    const [row] = await db.select({
      pending: sql<boolean>`${heartbeatRuns.status} <> 'running' or ${runtimeSettlementPending(companyId, runId)}`,
    }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId)));
    return row?.pending ?? true;
  }

  async function findLatestDecision(companyId: string, runId: string, now: Date) {
    const [quietUntilRows, dismissedRows] = await Promise.all([
      db
        .select({
          decision: heartbeatRunWatchdogDecisions.decision,
          snoozedUntil: heartbeatRunWatchdogDecisions.snoozedUntil,
        })
        .from(heartbeatRunWatchdogDecisions)
        .where(
          and(
            eq(heartbeatRunWatchdogDecisions.companyId, companyId),
            eq(heartbeatRunWatchdogDecisions.runId, runId),
            inArray(heartbeatRunWatchdogDecisions.decision, ["snooze", "continue"]),
            gt(heartbeatRunWatchdogDecisions.snoozedUntil, now),
          ),
        )
        .orderBy(desc(heartbeatRunWatchdogDecisions.createdAt))
        .limit(1),
      db
        .select({ id: heartbeatRunWatchdogDecisions.id })
        .from(heartbeatRunWatchdogDecisions)
        .where(
          and(
            eq(heartbeatRunWatchdogDecisions.companyId, companyId),
            eq(heartbeatRunWatchdogDecisions.runId, runId),
            eq(heartbeatRunWatchdogDecisions.decision, "dismissed_false_positive"),
          ),
        )
        .limit(1),
    ]);
    const quietUntilRow = quietUntilRows[0];
    return {
      dismissedFalsePositive: dismissedRows.length > 0,
      quietUntilDecision: quietUntilRow && quietUntilRow.snoozedUntil
        ? { decision: quietUntilRow.decision as "snooze" | "continue", snoozedUntil: quietUntilRow.snoozedUntil }
        : null,
    };
  }

  function selectEvaluationIssueSnapshot() {
    return {
      id: issues.id,
      identifier: issues.identifier,
      status: issues.status,
      assigneeAgentId: issues.assigneeAgentId,
      companyId: issues.companyId,
      originKind: issues.originKind,
      originId: issues.originId,
      hiddenAt: issues.hiddenAt,
    };
  }

  async function findOpenStaleRunEvaluation(companyId: string, runId: string): Promise<EvaluationIssueSnapshot | null> {
    const [row] = await db
      .select(selectEvaluationIssueSnapshot())
      .from(issues)
      .where(
        and(
          eq(issues.companyId, companyId),
          eq(issues.originKind, STALE_ACTIVE_RUN_EVALUATION_ORIGIN_KIND),
          eq(issues.originId, runId),
          visibleIssueCondition(),
          notInArray(issues.status, ["done", "cancelled"]),
        ),
      )
      .limit(1);
    return row ?? null;
  }

  async function findEvaluationIssueById(companyId: string, issueId: string): Promise<EvaluationIssueSnapshot | null> {
    const [row] = await db
      .select(selectEvaluationIssueSnapshot())
      .from(issues)
      .where(and(eq(issues.id, issueId), eq(issues.companyId, companyId)))
      .limit(1);
    return row ?? null;
  }

  async function findLatestSameRunTerminalEvidence(companyId: string, input: {
    runId: string;
    sourceIssueId: string;
    sourceIssueStatus: string;
    evidenceAfter: Date | null;
  }) {
    if (!isTerminalIssueStatus(input.sourceIssueStatus)) return null;
    const activityPredicates = [
      eq(activityLog.companyId, companyId),
      eq(activityLog.runId, input.runId),
      eq(activityLog.action, "issue.updated"),
      eq(activityLog.entityType, "issue"),
      eq(activityLog.entityId, input.sourceIssueId),
      sql`${activityLog.details} ->> 'status' = ${input.sourceIssueStatus}`,
    ];
    if (input.evidenceAfter) {
      activityPredicates.push(gte(activityLog.createdAt, input.evidenceAfter));
    }

    const activity = await db
      .select({ id: activityLog.id, createdAt: activityLog.createdAt, action: activityLog.action })
      .from(activityLog)
      .where(and(...activityPredicates))
      .orderBy(desc(activityLog.createdAt))
      .limit(1)
      .then((rows) => rows[0] ?? null);

    return activity ? { kind: "activity" as const, id: activity.id, createdAt: activity.createdAt, action: activity.action } : null;
  }

  async function findSourceIssue(companyId: string, issueId: string): Promise<SourceIssueSnapshot | null> {
    const [row] = await db
      .select({ id: issues.id, identifier: issues.identifier, status: issues.status, originKind: issues.originKind })
      .from(issues)
      .where(and(eq(issues.companyId, companyId), eq(issues.id, issueId), visibleIssueCondition()))
      .limit(1);
    return row ? { ...row, isRecoveryOriginKind: isRecoveryOriginKind(row.originKind) } : null;
  }

  async function findRunningAgent(companyId: string, agentId: string) {
    const [row] = await db
      .select({ id: agents.id, companyId: agents.companyId, adapterType: agents.adapterType })
      .from(agents)
      .where(and(eq(agents.id, agentId), eq(agents.companyId, companyId)))
      .limit(1);
    return row ?? null;
  }

  async function recordDecision(companyId: string, input: RecordDecisionInput): Promise<WatchdogDecisionRecord> {
    const [row] = await db
      .insert(heartbeatRunWatchdogDecisions)
      .values({
        companyId,
        runId: input.runId,
        evaluationIssueId: input.evaluationIssueId,
        decision: input.decision,
        snoozedUntil: input.snoozedUntil,
        reason: input.reason,
        createdByAgentId: input.createdByAgentId,
        createdByUserId: input.createdByUserId,
        createdByRunId: input.createdByRunId,
      })
      .returning();

    await logActivity(db, {
      companyId,
      actorType: input.actor.type === "agent" ? "agent" : "user",
      actorId: input.actor.type === "agent"
        ? input.actor.agentId ?? "agent"
        : input.actor.type === "board"
          ? input.actor.userId ?? "board"
          : "unknown",
      agentId: input.actor.type === "agent" ? input.actor.agentId ?? null : null,
      runId: input.runId,
      action: input.decision === "snooze" ? "heartbeat.watchdog_snoozed" : "heartbeat.watchdog_decision_recorded",
      entityType: "heartbeat_run",
      entityId: input.runId,
      details: {
        source: "recovery.record_watchdog_decision",
        decision: input.decision,
        evaluationIssueId: input.evaluationIssueId,
        snoozedUntil: input.snoozedUntil?.toISOString() ?? null,
        reason: input.reason,
      },
    });

    return { ...row, decision: row.decision as WatchdogDecisionRecord["decision"] };
  }

  async function foldSourceResolvedRun(companyId: string, input: FoldSourceResolvedRunInput): Promise<FoldOutcome> {
    const finalRunStatus = input.sourceIssue.status === "cancelled" ? "cancelled" : "succeeded";
    const postCommitIssueActions: IssuePostCommitAction[] = [];
    const resultJson = {
      ...parseObject(input.run.resultJson),
      sourceResolvedWatchdogFold: {
        sourceIssueId: input.sourceIssue.id,
        sourceIssueIdentifier: input.sourceIssue.identifier,
        sourceIssueStatus: input.sourceIssue.status,
        sameRunEvidenceKind: input.evidence.kind,
        sameRunEvidenceId: input.evidence.id,
        sameRunEvidenceAt: input.evidence.createdAt.toISOString(),
        silenceStartedAt: input.silenceStartedAt?.toISOString() ?? null,
        silenceAgeMs: input.silenceAgeMs,
        evaluationIssueId: input.existingEvaluation?.id ?? null,
        evaluationIssueIdentifier: input.existingEvaluation?.identifier ?? null,
        cleanup: input.cleanup,
      },
    };

    const transactionResult = await db.transaction(async (tx) => {
      await notifyInstructionCollection(tx, { companyId, runId: input.run.id });
      await notifyExternalAgentSettlement(tx, { companyId, runId: input.run.id });
      await notifyCancelledWorkspacePreservation(tx, { companyId, runId: input.run.id });
      await notifyUnsafeWorkspaceExportRecovery(tx, { companyId, issueId: input.sourceIssue.id });
      await notifyNativeStatusReassessment(tx, { companyId, issueId: input.sourceIssue.id });
      await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
      const [updatedRun] = await tx
        .update(heartbeatRuns)
        .set({
          status: finalRunStatus,
          finishedAt: input.now,
          error: null,
          errorCode: null,
          resultJson,
          updatedAt: input.now,
        })
        .where(and(
          eq(heartbeatRuns.id, input.run.id),
          eq(heartbeatRuns.companyId, companyId),
          eq(heartbeatRuns.status, "running"),
          sql`not ${runtimeSettlementPending(companyId, input.run.id)}`,
        ))
        .returning();
      if (!updatedRun) return null;

      if (input.run.wakeupRequestId) {
        await tx
          .update(agentWakeupRequests)
          .set({
            status: finalRunStatus === "succeeded" ? "completed" : "cancelled",
            finishedAt: input.now,
            error: null,
            updatedAt: input.now,
          })
          .where(and(
            eq(agentWakeupRequests.id, input.run.wakeupRequestId),
            eq(agentWakeupRequests.companyId, companyId),
            // A late success fold cannot restore cancelled assignment authority.
            ne(agentWakeupRequests.status, "cancelled"),
          ));
      }

      await tx
        .update(issues)
        .set({
          executionRunId: null,
          executionAgentNameKey: null,
          executionLockedAt: null,
          updatedAt: input.now,
        })
        .where(and(
          eq(issues.id, input.sourceIssue.id),
          eq(issues.companyId, companyId),
          eq(issues.executionRunId, input.run.id),
        ));

      if (input.existingEvaluation && !isTerminalIssueStatus(input.existingEvaluation.status)) {
        const updatedEvaluation = await issuesSvc.update(
          input.existingEvaluation.id,
          { status: "done" },
          tx,
          undefined,
          postCommitIssueActions,
        );
        if (!updatedEvaluation) {
          throw new Error("Evaluation issue disappeared during source-resolved watchdog fold");
        }
        await issuesSvc.addComment(input.existingEvaluation.id, [
          "Source-resolved watchdog fold.",
          "",
          `- Source issue: ${input.sourceIssue.identifier ?? input.sourceIssue.id}`,
          `- Run: \`${input.run.id}\``,
          `- Same-run evidence: \`${input.evidence.kind}:${input.evidence.id}\` at ${input.evidence.createdAt.toISOString()}`,
          "- Outcome: false positive; the source issue already reached a terminal disposition from this run.",
        ].join("\n"), { runId: input.run.id }, undefined, tx);
      }

      const activeRecoveryAction = await recoveryActionsSvc.getActiveForIssue(
        companyId,
        input.sourceIssue.id,
        tx,
      );
      if (activeRecoveryAction?.kind === "active_run_watchdog") {
        await recoveryActionsSvc.resolveActiveForIssue({
          companyId,
          sourceIssueId: input.sourceIssue.id,
          actionId: activeRecoveryAction.id,
          status: "resolved",
          outcome: "false_positive",
          resolutionNote: "Source issue reached a terminal disposition through durable same-run activity; watchdog folded as source-resolved.",
        }, tx);
      }

      const [decision] = await tx
        .insert(heartbeatRunWatchdogDecisions)
        .values({
          companyId,
          runId: input.run.id,
          evaluationIssueId: input.existingEvaluation?.id ?? null,
          decision: "dismissed_false_positive",
          reason: "Source issue already reached a terminal disposition through durable same-run activity.",
          createdByRunId: input.run.id,
        })
        .returning();

      await appendHeartbeatRunEvent(tx as unknown as Db, {
        companyId,
        runId: input.run.id,
        agentId: input.run.agentId,
        eventType: "lifecycle",
        stream: "system",
        level: input.cleanup.outcome === "failed" ? "warn" : "info",
        message: "Source-resolved watchdog fold finalized stale active run",
        payload: resultJson.sourceResolvedWatchdogFold,
      });

      await logActivity(tx as unknown as Db, {
        companyId,
        actorType: "system",
        actorId: "system",
        agentId: input.run.agentId,
        runId: input.run.id,
        action: "heartbeat.output_stale_source_resolved",
        entityType: "heartbeat_run",
        entityId: input.run.id,
        details: {
          source: "recovery.scan_silent_active_runs",
          sourceIssueId: input.sourceIssue.id,
          sourceIssueIdentifier: input.sourceIssue.identifier,
          sourceIssueStatus: input.sourceIssue.status,
          evaluationIssueId: input.existingEvaluation?.id ?? null,
          watchdogDecisionId: decision.id,
          sameRunEvidenceKind: input.evidence.kind,
          sameRunEvidenceId: input.evidence.id,
          sameRunEvidenceAt: input.evidence.createdAt.toISOString(),
          cleanup: input.cleanup,
        },
      });

      const [runningCountRow] = await tx
        .select({ count: sql<number>`count(*)::int` })
        .from(heartbeatRuns)
        .where(and(
          eq(heartbeatRuns.companyId, companyId),
          eq(heartbeatRuns.agentId, input.run.agentId),
          eq(heartbeatRuns.status, "running"),
        ));
      const runningCount = Number(runningCountRow?.count ?? 0);
      const nextAgentStatus = runningCount > 0 ? "running" : "idle";
      await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
      await tx
        .update(agents)
        .set({ status: nextAgentStatus, lastHeartbeatAt: new Date(), updatedAt: new Date() })
        .where(and(
          eq(agents.id, input.run.agentId),
          eq(agents.companyId, companyId),
          notInArray(agents.status, ["paused", "terminated"]),
          eq(agents.lifecycleState, "ready"),
        ));

      return updatedRun;
    });

    if (!transactionResult) return { kind: "stale" };
    const finalizedRun = transactionResult;

    await executeIssuePostCommitActions(db, postCommitIssueActions);

    // Telemetry is best-effort background work; it must not delay the
    // watchdog fold's caller, so fire it and do not await it.
    void emitAgentTaskRun(db, finalizedRun);

    return { kind: "folded", evaluationIssueId: input.existingEvaluation?.id ?? null };
  }

  return {
    findCandidateSilentRuns,
    findRunForCompany,
    isRuntimeSettlementPending,
    findLatestDecision,
    findOpenStaleRunEvaluation,
    findEvaluationIssueById,
    findLatestSameRunTerminalEvidence,
    findSourceIssue,
    findRunningAgent,
    recordDecision,
    foldSourceResolvedRun,
  };
}