diff --git a/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts b/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts index e6c9c807d02..0a33094db60 100644 --- a/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts +++ b/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts @@ -10,6 +10,7 @@ import { heartbeatRuns, issueComments, issueRelations, + issueTreeHolds, issues, projects, projectWorkspaces, @@ -229,6 +230,33 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => { expect(escalations).toHaveLength(0); }); + it("suppresses liveness escalation when the source issue is under an active pause hold", async () => { + await enableAutoRecovery(); + const { companyId, blockedIssueId } = await seedBlockedChain(); + + await db.insert(issueTreeHolds).values({ + companyId, + rootIssueId: blockedIssueId, + mode: "pause", + status: "active", + reason: "pause liveness recovery subtree", + releasePolicy: { strategy: "manual" }, + }); + + const result = await heartbeatService(db).reconcileIssueGraphLiveness(); + + expect(result.findings).toBe(1); + expect(result.escalationsCreated).toBe(0); + expect(result.existingEscalations).toBe(0); + expect(result.skipped).toBe(1); + + const escalations = await db + .select() + .from(issues) + .where(and(eq(issues.companyId, companyId), eq(issues.originKind, "harness_liveness_escalation"))); + expect(escalations).toHaveLength(0); + }); + it("treats an active executionRunId on the leaf blocker as a live execution path", async () => { await enableAutoRecovery(); const { companyId, managerId, blockedIssueId, blockerIssueId } = await seedBlockedChain(); diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index b94e1154cc1..47ab0e2a670 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -912,6 +912,48 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(comments[0]?.body).toContain(`Recovery issue: [${recovery.identifier}]`); }); + it("does not block paused-tree work when immediate continuation recovery is suppressed by the hold", async () => { + const { companyId, agentId, runId, issueId } = await seedRunFixture({ + agentStatus: "idle", + processPid: 999_999_999, + processLossRetryCount: 1, + }); + await db.insert(issueTreeHolds).values({ + companyId, + rootIssueId: issueId, + mode: "pause", + status: "active", + reason: "pause immediate recovery subtree", + releasePolicy: { strategy: "manual" }, + }); + const heartbeat = heartbeatService(db); + + const result = await heartbeat.reapOrphanedRuns(); + expect(result.reaped).toBe(1); + expect(result.runIds).toEqual([runId]); + + const runs = await db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.agentId, agentId)); + expect(runs).toHaveLength(1); + expect(runs[0]?.status).toBe("failed"); + + const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null); + expect(issue?.status).toBe("in_progress"); + expect(issue?.executionRunId).toBeNull(); + expect(issue?.checkoutRunId).toBe(runId); + + const recoveryIssues = await db + .select() + .from(issues) + .where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery"))); + expect(recoveryIssues).toHaveLength(0); + + const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId)); + expect(comments).toHaveLength(0); + }); + it("schedules a bounded retry for codex transient upstream failures instead of blocking the issue immediately", async () => { mockAdapterExecute.mockResolvedValueOnce({ exitCode: 1, diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index a0547ac4e0e..a432fec602f 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -113,6 +113,7 @@ import { findExistingRunLivenessContinuationWake, readContinuationAttempt, } from "./recovery/index.js"; +import { isAutomaticRecoverySuppressedByPauseHold } from "./recovery/pause-hold-guard.js"; import { recoveryService } from "./recovery/service.js"; import { redactCurrentUserText, redactCurrentUserValue } from "../log-redaction.js"; import { @@ -5878,6 +5879,10 @@ export function heartbeatService(db: Db) { return { kind: "released" as const }; } + if (await isAutomaticRecoverySuppressedByPauseHold(db, issue.companyId, issue.id)) { + return { kind: "released" as const }; + } + const shouldBlockImmediately = !recoveryAgentInvokable || !recoveryAgent || diff --git a/server/src/services/recovery/pause-hold-guard.ts b/server/src/services/recovery/pause-hold-guard.ts new file mode 100644 index 00000000000..6445341b58c --- /dev/null +++ b/server/src/services/recovery/pause-hold-guard.ts @@ -0,0 +1,11 @@ +import type { Db } from "@paperclipai/db"; +import { issueTreeControlService } from "../issue-tree-control.js"; + +export async function isAutomaticRecoverySuppressedByPauseHold( + db: Db, + companyId: string, + issueId: string, +) { + const activePauseHold = await issueTreeControlService(db).getActivePauseHoldGate(companyId, issueId); + return Boolean(activePauseHold); +} diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 87b687e9397..2c39044efab 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -30,6 +30,7 @@ import { classifyIssueGraphLiveness, type IssueLivenessFinding, } from "./issue-graph-liveness.js"; +import { isAutomaticRecoverySuppressedByPauseHold } from "./pause-hold-guard.js"; const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const; const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = ["failed", "cancelled", "timed_out"] as const; @@ -1437,6 +1438,11 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) continue; } + if (await isAutomaticRecoverySuppressedByPauseHold(db, issue.companyId, issue.id)) { + result.skipped += 1; + continue; + } + const latestRun = await getLatestIssueRun(issue.companyId, issue.id); if (issue.status === "todo") { if (!latestRun || latestRun.status === "succeeded") { @@ -1907,6 +1913,9 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) .where(eq(issues.id, input.finding.issueId)) .then((rows) => rows[0] ?? null); if (!issue || issue.companyId !== input.finding.companyId) return { kind: "skipped" as const }; + if (await isAutomaticRecoverySuppressedByPauseHold(db, issue.companyId, issue.id)) { + return { kind: "skipped" as const }; + } const recoveryIssue = await db .select()