Suppress recovery under pause holds

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts

@@ -10,6 +10,7 @@ import {
   heartbeatRuns,
   issueComments,
   issueRelations,
+  issueTreeHolds,
   issues,
   projects,
   projectWorkspaces,
@@ -229,6 +230,33 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => {
     expect(escalations).toHaveLength(0);
   });
 
+  it("suppresses liveness escalation when the source issue is under an active pause hold", async () => {
+    await enableAutoRecovery();
+    const { companyId, blockedIssueId } = await seedBlockedChain();
+
+    await db.insert(issueTreeHolds).values({
+      companyId,
+      rootIssueId: blockedIssueId,
+      mode: "pause",
+      status: "active",
+      reason: "pause liveness recovery subtree",
+      releasePolicy: { strategy: "manual" },
+    });
+
+    const result = await heartbeatService(db).reconcileIssueGraphLiveness();
+
+    expect(result.findings).toBe(1);
+    expect(result.escalationsCreated).toBe(0);
+    expect(result.existingEscalations).toBe(0);
+    expect(result.skipped).toBe(1);
+
+    const escalations = await db
+      .select()
+      .from(issues)
+      .where(and(eq(issues.companyId, companyId), eq(issues.originKind, "harness_liveness_escalation")));
+    expect(escalations).toHaveLength(0);
+  });
+
   it("treats an active executionRunId on the leaf blocker as a live execution path", async () => {
     await enableAutoRecovery();
     const { companyId, managerId, blockedIssueId, blockerIssueId } = await seedBlockedChain();

server/src/__tests__/heartbeat-process-recovery.test.ts

@@ -912,6 +912,48 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
     expect(comments[0]?.body).toContain(`Recovery issue: [${recovery.identifier}]`);
   });
 
+  it("does not block paused-tree work when immediate continuation recovery is suppressed by the hold", async () => {
+    const { companyId, agentId, runId, issueId } = await seedRunFixture({
+      agentStatus: "idle",
+      processPid: 999_999_999,
+      processLossRetryCount: 1,
+    });
+    await db.insert(issueTreeHolds).values({
+      companyId,
+      rootIssueId: issueId,
+      mode: "pause",
+      status: "active",
+      reason: "pause immediate recovery subtree",
+      releasePolicy: { strategy: "manual" },
+    });
+    const heartbeat = heartbeatService(db);
+
+    const result = await heartbeat.reapOrphanedRuns();
+    expect(result.reaped).toBe(1);
+    expect(result.runIds).toEqual([runId]);
+
+    const runs = await db
+      .select()
+      .from(heartbeatRuns)
+      .where(eq(heartbeatRuns.agentId, agentId));
+    expect(runs).toHaveLength(1);
+    expect(runs[0]?.status).toBe("failed");
+
+    const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null);
+    expect(issue?.status).toBe("in_progress");
+    expect(issue?.executionRunId).toBeNull();
+    expect(issue?.checkoutRunId).toBe(runId);
+
+    const recoveryIssues = await db
+      .select()
+      .from(issues)
+      .where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery")));
+    expect(recoveryIssues).toHaveLength(0);
+
+    const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId));
+    expect(comments).toHaveLength(0);
+  });
+
   it("schedules a bounded retry for codex transient upstream failures instead of blocking the issue immediately", async () => {
     mockAdapterExecute.mockResolvedValueOnce({
       exitCode: 1,

server/src/services/heartbeat.ts

@@ -113,6 +113,7 @@ import {
   findExistingRunLivenessContinuationWake,
   readContinuationAttempt,
 } from "./recovery/index.js";
+import { isAutomaticRecoverySuppressedByPauseHold } from "./recovery/pause-hold-guard.js";
 import { recoveryService } from "./recovery/service.js";
 import { redactCurrentUserText, redactCurrentUserValue } from "../log-redaction.js";
 import {
@@ -5878,6 +5879,10 @@ export function heartbeatService(db: Db) {
         return { kind: "released" as const };
       }
 
+      if (await isAutomaticRecoverySuppressedByPauseHold(db, issue.companyId, issue.id)) {
+        return { kind: "released" as const };
+      }
+
       const shouldBlockImmediately =
         !recoveryAgentInvokable ||
         !recoveryAgent ||

server/src/services/recovery/pause-hold-guard.ts

@@ -0,0 +1,11 @@
+import type { Db } from "@paperclipai/db";
+import { issueTreeControlService } from "../issue-tree-control.js";
+
+export async function isAutomaticRecoverySuppressedByPauseHold(
+  db: Db,
+  companyId: string,
+  issueId: string,
+) {
+  const activePauseHold = await issueTreeControlService(db).getActivePauseHoldGate(companyId, issueId);
+  return Boolean(activePauseHold);
+}

server/src/services/recovery/service.ts

@@ -30,6 +30,7 @@ import {
   classifyIssueGraphLiveness,
   type IssueLivenessFinding,
 } from "./issue-graph-liveness.js";
+import { isAutomaticRecoverySuppressedByPauseHold } from "./pause-hold-guard.js";
 
 const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const;
 const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = ["failed", "cancelled", "timed_out"] as const;
@@ -1437,6 +1438,11 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
         continue;
       }
 
+      if (await isAutomaticRecoverySuppressedByPauseHold(db, issue.companyId, issue.id)) {
+        result.skipped += 1;
+        continue;
+      }
+
       const latestRun = await getLatestIssueRun(issue.companyId, issue.id);
       if (issue.status === "todo") {
         if (!latestRun || latestRun.status === "succeeded") {
@@ -1907,6 +1913,9 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup })
       .where(eq(issues.id, input.finding.issueId))
       .then((rows) => rows[0] ?? null);
     if (!issue || issue.companyId !== input.finding.companyId) return { kind: "skipped" as const };
+    if (await isAutomaticRecoverySuppressedByPauseHold(db, issue.companyId, issue.id)) {
+      return { kind: "skipped" as const };
+    }
 
     const recoveryIssue = await db
       .select()