fix(heartbeat): finalize interrupted dead runs

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/heartbeat-process-recovery.test.ts

@@ -584,13 +584,26 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
     return { environmentId, leaseId };
   }
 
-  async function seedDeadRunFinalizeBarrierFixture(input?: { runStatus?: "running" | "failed" }) {
+  async function seedDeadRunFinalizeBarrierFixture(input?: {
+    runStatus?: "running" | "interrupted" | "failed";
+  }) {
+    const runStatus = input?.runStatus ?? "running";
+    const runErrorCode = runStatus === "failed"
+      ? "process_lost"
+      : runStatus === "interrupted"
+        ? "server_shutdown_interrupted"
+        : null;
+    const runError = runStatus === "failed"
+      ? "Process lost during a prior server lifetime"
+      : runStatus === "interrupted"
+        ? "Run interrupted during graceful server shutdown"
+        : null;
     const seeded = await seedRunFixture({
       agentStatus: "running",
       adapterType: "test",
-      runStatus: input?.runStatus ?? "running",
-      runErrorCode: input?.runStatus === "failed" ? "process_lost" : null,
-      runError: input?.runStatus === "failed" ? "Process lost during a prior server lifetime" : null,
+      runStatus,
+      runErrorCode,
+      runError,
     });
     const projectId = randomUUID();
     const executionWorkspaceId = randomUUID();
@@ -631,7 +644,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
           issueId: seeded.issueId,
           executionWorkspaceId,
         },
-        ...(input?.runStatus === "failed" ? { finishedAt: now, updatedAt: now } : {}),
+        ...(runStatus !== "running" ? { finishedAt: now, updatedAt: now } : {}),
       })
       .where(eq(heartbeatRuns.id, seeded.runId));
 
@@ -1489,6 +1502,41 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
     });
   });
 
+  it("releases a finalize barrier owned by an interrupted run", async () => {
+    const fixture = await seedDeadRunFinalizeBarrierFixture({ runStatus: "interrupted" });
+
+    const result = await heartbeatService(db).reconcileTerminalRunWorkspaceFinalizes("startup_sweep");
+
+    expect(result).toMatchObject({
+      checked: 1,
+      created: 1,
+      wakesHealed: 1,
+      runIds: [fixture.runId],
+    });
+    const syntheticFinalize = await db
+      .select()
+      .from(workspaceOperations)
+      .where(
+        and(
+          eq(workspaceOperations.heartbeatRunId, fixture.runId),
+          eq(workspaceOperations.phase, "workspace_finalize"),
+        ),
+      )
+      .then((rows) => rows[0] ?? null);
+    expect(syntheticFinalize).toMatchObject({
+      status: "succeeded",
+      metadata: expect.objectContaining({
+        synthetic: true,
+        reason: "server_shutdown_interrupted",
+        terminalRunStatus: "interrupted",
+      }),
+    });
+    await expect(issueService(db).getDependencyReadiness(fixture.dependentIssueId)).resolves.toMatchObject({
+      isDependencyReady: true,
+      pendingFinalizeBlockerIssueIds: [],
+    });
+  });
+
   it("does not mask an explicit failed workspace finalize with a synthetic success", async () => {
     const fixture = await seedDeadRunFinalizeBarrierFixture({ runStatus: "failed" });
     const failedAt = new Date("2026-03-19T00:02:00.000Z");

server/src/services/heartbeat.ts

@@ -291,6 +291,10 @@ const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_r
 const CANCELLABLE_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const;
 const HEARTBEAT_RUN_TERMINAL_STATUSES = ["succeeded", "interrupted", "failed", "cancelled", "timed_out"] as const;
 const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = ["failed", "cancelled", "timed_out"] as const;
+const WORKSPACE_FINALIZE_RECOVERABLE_TERMINAL_STATUSES = [
+  "interrupted",
+  ...UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES,
+] as const;
 const TIMER_ACTIONABLE_ISSUE_STATUSES = ["todo", "in_progress"] as const;
 export {
   ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS,
@@ -5102,8 +5106,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
 
     const reconciliation = (async () => {
       if (
-        !UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES.includes(
-          run.status as (typeof UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES)[number],
+        !WORKSPACE_FINALIZE_RECOVERABLE_TERMINAL_STATUSES.includes(
+          run.status as (typeof WORKSPACE_FINALIZE_RECOVERABLE_TERMINAL_STATUSES)[number],
         )
       ) {
         return { created: false, wakeHealed: 0 };
@@ -5306,7 +5310,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
           and(
             eq(heartbeatRuns.id, latestOperation.heartbeatRunId),
             eq(heartbeatRuns.companyId, blocker.companyId),
-            inArray(heartbeatRuns.status, [...UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES]),
+            inArray(heartbeatRuns.status, [...WORKSPACE_FINALIZE_RECOVERABLE_TERMINAL_STATUSES]),
           ),
         )
         .then((rows) => rows[0] ?? null);