fix: preserve workspace finalization during watchdog recovery (#16247)

Keep task completion separate from runtime and workspace settlement. Permit verified board file-only repair of successful legacy retained-source runs without authorizing replay.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/heartbeat-active-run-output-watchdog.test.ts

@@ -7,6 +7,7 @@ import {
   agentWakeupRequests,
   companies,
   createDb,
+  environmentLeases,
   heartbeatRunEvents,
   heartbeatRunWatchdogDecisions,
   heartbeatRuns,
@@ -14,6 +15,7 @@ import {
   issueRecoveryActions,
   issueRelations,
   issues,
+  workspaceOperations,
 } from "@paperclipai/db";
 import {
   getEmbeddedPostgresTestSupport,
@@ -21,6 +23,7 @@ import {
 } from "./helpers/embedded-postgres.js";
 import { truncateTablesWithDeadlockRetry } from "./helpers/truncate-with-deadlock-retry.js";
 import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js";
+import { createPostgresWatchdogAdapter } from "../modules/active-run-watchdog/adapters/postgres.js";
 import {
   ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS,
   ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS,
@@ -446,6 +449,75 @@ describeEmbeddedPostgres("active-run output watchdog", () => {
     ))).toHaveLength(0);
   });
 
+  it.each(["remote_lease", "workspace_finalize", "failed_finalize", "native_owner", "restore_failed"])(
+    "leaves a terminal task's run with its lifecycle owner while %s is unsettled",
+    async pending => {
+      const now = new Date("2026-04-22T20:00:00.000Z");
+      const seeded = await seedRunningRun({ now, ageMs: ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS + 60_000,
+        sourceStatus: "done", sameRunTerminalEvidence: true });
+      if (pending === "remote_lease") {
+        await db.insert(environmentLeases).values({ companyId: seeded.companyId, heartbeatRunId: seeded.runId,
+          issueId: seeded.issueId, status: "active", provider: "daytona", providerLeaseId: "sandbox-with-unsaved-files" });
+      } else if (pending === "workspace_finalize" || pending === "failed_finalize") {
+        await db.insert(workspaceOperations).values({ companyId: seeded.companyId, heartbeatRunId: seeded.runId,
+          issueId: seeded.issueId, phase: "workspace_finalize", status: pending === "failed_finalize" ? "failed" : "running" });
+      } else if (pending === "native_owner") {
+        await db.update(heartbeatRuns).set({ runtimeMode: "native" }).where(eq(heartbeatRuns.id, seeded.runId));
+      } else {
+        await db.update(heartbeatRuns).set({ resultJson: { workspaceRestoreFailure: "restore_failed" } })
+          .where(eq(heartbeatRuns.id, seeded.runId));
+      }
+      const { recovery, enqueueWakeup } = createRecovery();
+      expect(await recovery.scanSilentActiveRuns({ now, companyId: seeded.companyId }))
+        .toMatchObject({ folded: 0, created: 0, skipped: 1 });
+      const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.runId));
+      const [task] = await db.select().from(issues).where(eq(issues.id, seeded.issueId));
+      const [agent] = await db.select().from(agents).where(eq(agents.id, seeded.coderId));
+      expect(run.status).toBe("running");
+      expect(task).toMatchObject({ status: "done", executionRunId: seeded.runId });
+      expect(agent.status).toBe("running");
+      expect(enqueueWakeup).not.toHaveBeenCalled();
+      expect(await db.select().from(heartbeatRunWatchdogDecisions).where(eq(heartbeatRunWatchdogDecisions.runId, seeded.runId)))
+        .toHaveLength(0);
+    },
+  );
+
+  it("folds stale bookkeeping after remote release and successful workspace finalization", async () => {
+    const now = new Date("2026-04-22T20:00:00.000Z");
+    const seeded = await seedRunningRun({ now, ageMs: ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS + 60_000,
+      sourceStatus: "done", sameRunTerminalEvidence: true });
+    await db.insert(environmentLeases).values({ companyId: seeded.companyId, heartbeatRunId: seeded.runId,
+      issueId: seeded.issueId, status: "released", releasedAt: now, cleanupStatus: "success",
+      provider: "daytona", providerLeaseId: "finished-sandbox" });
+    await db.insert(workspaceOperations).values({ companyId: seeded.companyId, heartbeatRunId: seeded.runId,
+      issueId: seeded.issueId, phase: "workspace_finalize", status: "succeeded", finishedAt: now });
+    expect(await createRecovery().recovery.scanSilentActiveRuns({ now, companyId: seeded.companyId }))
+      .toMatchObject({ folded: 1, created: 0 });
+  });
+
+  it("rechecks remote ownership when a lease appears after the initial fold check", async () => {
+    const now = new Date("2026-04-22T20:00:00.000Z");
+    const seeded = await seedRunningRun({ now, ageMs: ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS + 60_000,
+      sourceStatus: "done", sameRunTerminalEvidence: true });
+    const adapter = createPostgresWatchdogAdapter(db);
+    const run = await adapter.findRunForCompany(seeded.companyId, seeded.runId);
+    expect(await adapter.isRuntimeSettlementPending(seeded.companyId, seeded.runId)).toBe(false);
+    await db.insert(environmentLeases).values({ companyId: seeded.companyId, heartbeatRunId: seeded.runId,
+      issueId: seeded.issueId, status: "active", provider: "daytona", providerLeaseId: "late-owned-sandbox" });
+    expect(await adapter.foldSourceResolvedRun(seeded.companyId, {
+      run: run!, sourceIssue: { id: seeded.issueId, identifier: `${seeded.issuePrefix}-1`, status: "done",
+        originKind: "manual", isRecoveryOriginKind: false },
+      evidence: { kind: "activity", id: randomUUID(), action: "issue.updated", createdAt: now },
+      existingEvaluation: null, silenceStartedAt: seeded.startedAt, silenceAgeMs: 16 * 60 * 1000,
+      cleanup: { attempted: false, outcome: "no_process_metadata", adapterType: "grok_local" }, now,
+    })).toEqual({ kind: "stale" });
+    const [persisted] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.runId));
+    expect(persisted.status).toBe("running");
+    expect(persisted.resultJson).toBeNull();
+    expect(await db.select().from(heartbeatRunWatchdogDecisions).where(eq(heartbeatRunWatchdogDecisions.runId, seeded.runId)))
+      .toHaveLength(0);
+  });
+
   it("folds existing legacy evaluation and recovery rows idempotently", async () => {
     const now = new Date("2026-04-22T20:00:00.000Z");
     const seeded = await seedRunningRun({

server/src/__tests__/workspace-restore-repair-route.test.ts

@@ -117,8 +117,12 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
     expect(await db.select().from(heartbeatRuns).where(inArray(heartbeatRuns.id, [f.sourceRunId, f.currentRunId]))).toHaveLength(2);
   }
 
-  it.each(["reassigned", "conversation_reset", "unassigned"])("records board repair after %s without restarting or changing the current task", async scenario => {
+  it.each(["reassigned", "conversation_reset", "unassigned", "successful_source"])("records board repair after %s without restarting or changing the current task", async scenario => {
     const f = await seed({ conversationReset: scenario === "conversation_reset", noCurrentOwner: scenario === "unassigned" });
+    if (scenario === "successful_source") {
+      await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, f.sourceRunId));
+    }
+    const [sourceBefore] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.sourceRunId));
     const { app, wake } = appFor();
     expect(await getExecutionBlocker(db, f.companyId, f.issueId)).not.toBeNull();
     await request(app).post(`/api/issues/${f.issueId}/recovery-actions/resolve`).send(f.body).expect(200);
@@ -137,6 +141,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
     expect(await db.select().from(activityLog).where(eq(activityLog.companyId, f.companyId))).toEqual(auditBefore);
     await assertSourceUnchanged(f);
     expect(wake).not.toHaveBeenCalled();
+    expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.sourceRunId)))[0]).toEqual(sourceBefore);
   });
 
   it.each(["blocked", "done", "cancelled", "backlog"])("records repair without changing the current %s task", async status => {