fix: preserve workspace finalization during watchdog recovery (#16247)

Keep task completion separate from runtime and workspace settlement. Permit verified board file-only repair of successful legacy retained-source runs without authorizing replay.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/heartbeat-active-run-output-watchdog.test.ts

@@ -7,6 +7,7 @@ import {
   agentWakeupRequests,
   companies,
   createDb,
+  environmentLeases,
   heartbeatRunEvents,
   heartbeatRunWatchdogDecisions,
   heartbeatRuns,
@@ -14,6 +15,7 @@ import {
   issueRecoveryActions,
   issueRelations,
   issues,
+  workspaceOperations,
 } from "@paperclipai/db";
 import {
   getEmbeddedPostgresTestSupport,
@@ -21,6 +23,7 @@ import {
 } from "./helpers/embedded-postgres.js";
 import { truncateTablesWithDeadlockRetry } from "./helpers/truncate-with-deadlock-retry.js";
 import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js";
+import { createPostgresWatchdogAdapter } from "../modules/active-run-watchdog/adapters/postgres.js";
 import {
   ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS,
   ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS,
@@ -446,6 +449,75 @@ describeEmbeddedPostgres("active-run output watchdog", () => {
     ))).toHaveLength(0);
   });
 
+  it.each(["remote_lease", "workspace_finalize", "failed_finalize", "native_owner", "restore_failed"])(
+    "leaves a terminal task's run with its lifecycle owner while %s is unsettled",
+    async pending => {
+      const now = new Date("2026-04-22T20:00:00.000Z");
+      const seeded = await seedRunningRun({ now, ageMs: ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS + 60_000,
+        sourceStatus: "done", sameRunTerminalEvidence: true });
+      if (pending === "remote_lease") {
+        await db.insert(environmentLeases).values({ companyId: seeded.companyId, heartbeatRunId: seeded.runId,
+          issueId: seeded.issueId, status: "active", provider: "daytona", providerLeaseId: "sandbox-with-unsaved-files" });
+      } else if (pending === "workspace_finalize" || pending === "failed_finalize") {
+        await db.insert(workspaceOperations).values({ companyId: seeded.companyId, heartbeatRunId: seeded.runId,
+          issueId: seeded.issueId, phase: "workspace_finalize", status: pending === "failed_finalize" ? "failed" : "running" });
+      } else if (pending === "native_owner") {
+        await db.update(heartbeatRuns).set({ runtimeMode: "native" }).where(eq(heartbeatRuns.id, seeded.runId));
+      } else {
+        await db.update(heartbeatRuns).set({ resultJson: { workspaceRestoreFailure: "restore_failed" } })
+          .where(eq(heartbeatRuns.id, seeded.runId));
+      }
+      const { recovery, enqueueWakeup } = createRecovery();
+      expect(await recovery.scanSilentActiveRuns({ now, companyId: seeded.companyId }))
+        .toMatchObject({ folded: 0, created: 0, skipped: 1 });
+      const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.runId));
+      const [task] = await db.select().from(issues).where(eq(issues.id, seeded.issueId));
+      const [agent] = await db.select().from(agents).where(eq(agents.id, seeded.coderId));
+      expect(run.status).toBe("running");
+      expect(task).toMatchObject({ status: "done", executionRunId: seeded.runId });
+      expect(agent.status).toBe("running");
+      expect(enqueueWakeup).not.toHaveBeenCalled();
+      expect(await db.select().from(heartbeatRunWatchdogDecisions).where(eq(heartbeatRunWatchdogDecisions.runId, seeded.runId)))
+        .toHaveLength(0);
+    },
+  );
+
+  it("folds stale bookkeeping after remote release and successful workspace finalization", async () => {
+    const now = new Date("2026-04-22T20:00:00.000Z");
+    const seeded = await seedRunningRun({ now, ageMs: ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS + 60_000,
+      sourceStatus: "done", sameRunTerminalEvidence: true });
+    await db.insert(environmentLeases).values({ companyId: seeded.companyId, heartbeatRunId: seeded.runId,
+      issueId: seeded.issueId, status: "released", releasedAt: now, cleanupStatus: "success",
+      provider: "daytona", providerLeaseId: "finished-sandbox" });
+    await db.insert(workspaceOperations).values({ companyId: seeded.companyId, heartbeatRunId: seeded.runId,
+      issueId: seeded.issueId, phase: "workspace_finalize", status: "succeeded", finishedAt: now });
+    expect(await createRecovery().recovery.scanSilentActiveRuns({ now, companyId: seeded.companyId }))
+      .toMatchObject({ folded: 1, created: 0 });
+  });
+
+  it("rechecks remote ownership when a lease appears after the initial fold check", async () => {
+    const now = new Date("2026-04-22T20:00:00.000Z");
+    const seeded = await seedRunningRun({ now, ageMs: ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS + 60_000,
+      sourceStatus: "done", sameRunTerminalEvidence: true });
+    const adapter = createPostgresWatchdogAdapter(db);
+    const run = await adapter.findRunForCompany(seeded.companyId, seeded.runId);
+    expect(await adapter.isRuntimeSettlementPending(seeded.companyId, seeded.runId)).toBe(false);
+    await db.insert(environmentLeases).values({ companyId: seeded.companyId, heartbeatRunId: seeded.runId,
+      issueId: seeded.issueId, status: "active", provider: "daytona", providerLeaseId: "late-owned-sandbox" });
+    expect(await adapter.foldSourceResolvedRun(seeded.companyId, {
+      run: run!, sourceIssue: { id: seeded.issueId, identifier: `${seeded.issuePrefix}-1`, status: "done",
+        originKind: "manual", isRecoveryOriginKind: false },
+      evidence: { kind: "activity", id: randomUUID(), action: "issue.updated", createdAt: now },
+      existingEvaluation: null, silenceStartedAt: seeded.startedAt, silenceAgeMs: 16 * 60 * 1000,
+      cleanup: { attempted: false, outcome: "no_process_metadata", adapterType: "grok_local" }, now,
+    })).toEqual({ kind: "stale" });
+    const [persisted] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.runId));
+    expect(persisted.status).toBe("running");
+    expect(persisted.resultJson).toBeNull();
+    expect(await db.select().from(heartbeatRunWatchdogDecisions).where(eq(heartbeatRunWatchdogDecisions.runId, seeded.runId)))
+      .toHaveLength(0);
+  });
+
   it("folds existing legacy evaluation and recovery rows idempotently", async () => {
     const now = new Date("2026-04-22T20:00:00.000Z");
     const seeded = await seedRunningRun({

server/src/__tests__/workspace-restore-repair-route.test.ts

@@ -117,8 +117,12 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
     expect(await db.select().from(heartbeatRuns).where(inArray(heartbeatRuns.id, [f.sourceRunId, f.currentRunId]))).toHaveLength(2);
   }
 
-  it.each(["reassigned", "conversation_reset", "unassigned"])("records board repair after %s without restarting or changing the current task", async scenario => {
+  it.each(["reassigned", "conversation_reset", "unassigned", "successful_source"])("records board repair after %s without restarting or changing the current task", async scenario => {
     const f = await seed({ conversationReset: scenario === "conversation_reset", noCurrentOwner: scenario === "unassigned" });
+    if (scenario === "successful_source") {
+      await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, f.sourceRunId));
+    }
+    const [sourceBefore] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.sourceRunId));
     const { app, wake } = appFor();
     expect(await getExecutionBlocker(db, f.companyId, f.issueId)).not.toBeNull();
     await request(app).post(`/api/issues/${f.issueId}/recovery-actions/resolve`).send(f.body).expect(200);
@@ -137,6 +141,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
     expect(await db.select().from(activityLog).where(eq(activityLog.companyId, f.companyId))).toEqual(auditBefore);
     await assertSourceUnchanged(f);
     expect(wake).not.toHaveBeenCalled();
+    expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.sourceRunId)))[0]).toEqual(sourceBefore);
   });
 
   it.each(["blocked", "done", "cancelled", "backlog"])("records repair without changing the current %s task", async status => {

server/src/modules/active-run-watchdog/adapters/postgres.ts

@@ -6,7 +6,9 @@ import {
   activityLog,
   heartbeatRunWatchdogDecisions,
   heartbeatRuns,
+  environmentLeases,
   issues,
+  workspaceOperations,
 } from "@paperclipai/db";
 import { parseObject } from "../../../adapters/utils.js";
 import { visibleIssueCondition } from "../../../services/issue-visibility.js";
@@ -65,6 +67,27 @@ function toRunSnapshot(row: typeof heartbeatRuns.$inferSelect): RunSnapshot {
   };
 }
 
+function runtimeSettlementPending(companyId: string, runId: string) {
+  return sql<boolean>`(
+    ${heartbeatRuns.runtimeMode} = 'native'
+    or coalesce(${heartbeatRuns.resultJson}->>'workspaceRestoreFailure', '') <> ''
+    or exists (
+      select 1 from ${environmentLeases}
+      where ${environmentLeases.companyId} = ${companyId}
+        and ${environmentLeases.heartbeatRunId} = ${runId}
+        and ${environmentLeases.providerLeaseId} is not null
+        and ${environmentLeases.status} in ('active', 'acquiring')
+    )
+    or exists (
+      select 1 from ${workspaceOperations}
+      where ${workspaceOperations.companyId} = ${companyId}
+        and ${workspaceOperations.heartbeatRunId} = ${runId}
+        and ${workspaceOperations.phase} = 'workspace_finalize'
+        and ${workspaceOperations.status} in ('running', 'failed')
+    )
+  )`;
+}
+
 export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & WatchdogWriter {
   const issuesSvc = issueService(db);
   const recoveryActionsSvc = issueRecoveryActionService(db);
@@ -119,6 +142,13 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch
     return row ? toRunSnapshot(row) : null;
   }
 
+  async function isRuntimeSettlementPending(companyId: string, runId: string): Promise<boolean> {
+    const [row] = await db.select({
+      pending: sql<boolean>`${heartbeatRuns.status} <> 'running' or ${runtimeSettlementPending(companyId, runId)}`,
+    }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId)));
+    return row?.pending ?? true;
+  }
+
   async function findLatestDecision(companyId: string, runId: string, now: Date) {
     const [quietUntilRows, dismissedRows] = await Promise.all([
       db
@@ -321,6 +351,7 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch
           eq(heartbeatRuns.id, input.run.id),
           eq(heartbeatRuns.companyId, companyId),
           eq(heartbeatRuns.status, "running"),
+          sql`not ${runtimeSettlementPending(companyId, input.run.id)}`,
         ))
         .returning();
       if (!updatedRun) return null;
@@ -477,6 +508,7 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch
   return {
     findCandidateSilentRuns,
     findRunForCompany,
+    isRuntimeSettlementPending,
     findLatestDecision,
     findOpenStaleRunEvaluation,
     findEvaluationIssueById,

server/src/modules/active-run-watchdog/application/ports.ts

@@ -53,6 +53,7 @@ export type RunningAgentSnapshot = {
 export interface WatchdogRunReader {
   findCandidateSilentRuns(input: FindCandidateSilentRunsInput): Promise<RunSnapshot[]>;
   findRunForCompany(companyId: string, runId: string): Promise<RunSnapshot | null>;
+  isRuntimeSettlementPending(companyId: string, runId: string): Promise<boolean>;
   findLatestDecision(companyId: string, runId: string, now: Date): Promise<FindLatestDecisionResult>;
   findOpenStaleRunEvaluation(companyId: string, runId: string): Promise<EvaluationIssueSnapshot | null>;
   findLatestSameRunTerminalEvidence(

server/src/modules/active-run-watchdog/application/use-cases.test.ts

@@ -38,6 +38,7 @@ describe("application", () => {
     return {
       findCandidateSilentRuns: async () => [],
       findRunForCompany: async () => null,
+      isRuntimeSettlementPending: async () => false,
       findLatestDecision: async () => ({ quietUntilDecision: null, dismissedFalsePositive: false }),
       findOpenStaleRunEvaluation: async () => null,
       findLatestSameRunTerminalEvidence: async () => null,
@@ -134,6 +135,27 @@ describe("application", () => {
   };
 
   describe("createFoldSourceResolvedRun", () => {
+    it("does not terminate a process or settle the run while its runtime owns finalization", async () => {
+      const calls: string[] = [];
+      const fold = createFoldSourceResolvedRun({
+        reader: makeReader({ isRuntimeSettlementPending: async (companyId, runId) => {
+          expect([companyId, runId]).toEqual(["company-1", "run-1"]);
+          return true;
+        } }),
+        processController: { cleanupRunProcess: async () => {
+          calls.push("cleanup");
+          return { attempted: false, outcome: "no_process_metadata", adapterType: "grok_local" };
+        } },
+        writer: { recordDecision: async () => { throw new Error("unexpected decision"); },
+          foldSourceResolvedRun: async () => { calls.push("fold"); return { kind: "folded", evaluationIssueId: null }; } },
+      });
+      expect(await fold({ run: makeRun({ processPid: 4242 }), runningAgentAdapterType: "grok_local",
+        sourceIssue: foldSourceIssue, evidence: foldEvidence, existingEvaluation: null,
+        silenceStartedAt: null, silenceAgeMs: null, now: new Date("2026-01-01T05:00:00Z") }))
+        .toEqual({ kind: "stale" });
+      expect(calls).toEqual([]);
+    });
+
     it("cleans up the process before the folding persistence call", async () => {
       const calls: string[] = [];
       const processController: RunProcessController = {
@@ -154,7 +176,7 @@ describe("application", () => {
           return { kind: "folded", evaluationIssueId: null };
         },
       };
-      const foldSourceResolvedRun = createFoldSourceResolvedRun({ writer, processController });
+      const foldSourceResolvedRun = createFoldSourceResolvedRun({ reader: makeReader(), writer, processController });
 
       const outcome = await foldSourceResolvedRun({
         run: makeRun({ processPid: 4242 }),
@@ -187,7 +209,7 @@ describe("application", () => {
         },
         foldSourceResolvedRun: async () => ({ kind: "stale" }),
       };
-      const foldSourceResolvedRun = createFoldSourceResolvedRun({ writer, processController });
+      const foldSourceResolvedRun = createFoldSourceResolvedRun({ reader: makeReader(), writer, processController });
 
       const outcome = await foldSourceResolvedRun({
         run: makeRun({ processPid: 4242 }),
@@ -396,7 +418,7 @@ describe("application", () => {
       foldSourceResolvedRun: async () => ({ kind: "folded", evaluationIssueId: null }),
       ...writerOverrides,
     };
-    return createFoldSourceResolvedRun({ writer, processController });
+    return createFoldSourceResolvedRun({ reader: makeReader(), writer, processController });
   }
 
   describe("createScanSilentActiveRuns", () => {

server/src/modules/active-run-watchdog/application/use-cases.ts

@@ -72,6 +72,7 @@ export function createBuildRunOutputSilence(deps: BuildRunOutputSilenceDeps) {
 }
 
 export type FoldSourceResolvedRunDeps = {
+  reader: Pick<WatchdogRunReader, "isRuntimeSettlementPending">;
   writer: WatchdogWriter;
   processController: RunProcessController;
 };
@@ -89,6 +90,9 @@ export type FoldSourceResolvedRunUseCaseInput = {
 
 export function createFoldSourceResolvedRun(deps: FoldSourceResolvedRunDeps) {
   return async function foldSourceResolvedRun(input: FoldSourceResolvedRunUseCaseInput): Promise<FoldOutcome> {
+    if (await deps.reader.isRuntimeSettlementPending(input.run.companyId, input.run.id)) {
+      return { kind: "stale" };
+    }
     const cleanup = await deps.processController.cleanupRunProcess({
       runId: input.run.id,
       adapterType: input.runningAgentAdapterType,

server/src/modules/active-run-watchdog/index.ts

@@ -24,6 +24,7 @@ export function createActiveRunWatchdog(db: Db, config: ActiveRunWatchdogConfig)
   const processController = createProcessAdapter();
 
   const foldSourceResolvedRun = createFoldSourceResolvedRun({
+    reader: postgresAdapter,
     writer: postgresAdapter,
     processController,
   });

server/src/services/execution-recovery-resolution.ts

@@ -65,13 +65,15 @@ export async function validateExecutionReconciliation(input: {
     review.currentParticipant.agentId === run?.agentId;
   const retainedWorkspace = hasRequiredWorkspaceRecovery(run?.resultJson);
   const repairOnly = input.workspaceRepairOnly === true && retainedWorkspace;
+  const terminalRepairSource = ["failed", "interrupted", "timed_out", "cancelled"].includes(run?.status ?? "") ||
+    (repairOnly && run?.runtimeMode === "legacy" && run.status === "succeeded");
   if (
     (input.workspaceRepairOnly && !retainedWorkspace) ||
     !run ||
     !task ||
     (!repairOnly && (task.assigneeAgentId !== agentId || (run.agentId !== agentId && !isCurrentReviewer))) ||
     (run.nativeIssueId ?? run.contextSnapshot?.issueId) !== issueId ||
-    !["failed", "interrupted", "timed_out", "cancelled"].includes(run.status)
+    !terminalRepairSource
   ) {
     throw conflict(
       "The recovery source or task owner changed. Inspect the current execution before continuing.",

server/src/services/workspace-restore-reconciliation.test.ts

@@ -80,6 +80,39 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
     await expect(validateExecutionReconciliation({ db, ...f })).resolves.toMatchObject({ id: f.sourceRunId });
   });
 
+  it("repairs retained files from a succeeded run without reopening or replaying its completed work", async () => {
+    const f = await seed();
+    await db.update(heartbeatRuns).set({ status: "succeeded", errorCode: null }).where(eq(heartbeatRuns.id, f.sourceRunId));
+    await db.update(issues).set({ status: "todo" }).where(eq(issues.id, f.issueId));
+    const repaired = await validateExecutionReconciliation({ db, ...f, workspaceRepairOnly: true });
+    const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId));
+    await markExecutionReconciliation(db, action, f.decision, "board", undefined, { workspaceRepairOnly: true });
+    expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull();
+    expect(repaired.status).toBe("succeeded");
+    const [recorded] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId));
+    expect(recorded.evidence.continuationDelivery).toBe("not_requested");
+    const [task] = await db.select().from(issues).where(eq(issues.id, f.issueId));
+    expect(task).toMatchObject({ status: "todo", assigneeAgentId: f.agentId, executionRunId: null });
+  });
+
+  it.each(["running", "queued", "succeeded"])("does not authorize a normal continuation of a %s retained run", async status => {
+    const f = await seed();
+    await db.update(heartbeatRuns).set({ status }).where(eq(heartbeatRuns.id, f.sourceRunId));
+    await expect(validateExecutionReconciliation({ db, ...f })).rejects.toThrow("source or task owner changed");
+  });
+
+  it.each(["running", "queued"])("does not accept file repair while the source run is %s", async status => {
+    const f = await seed();
+    await db.update(heartbeatRuns).set({ status }).where(eq(heartbeatRuns.id, f.sourceRunId));
+    await expect(validateExecutionReconciliation({ db, ...f, workspaceRepairOnly: true })).rejects.toThrow("source or task owner changed");
+  });
+
+  it("does not replace native finalization with legacy success repair", async () => {
+    const f = await seed();
+    await db.update(heartbeatRuns).set({ status: "succeeded", runtimeMode: "native" }).where(eq(heartbeatRuns.id, f.sourceRunId));
+    await expect(validateExecutionReconciliation({ db, ...f, workspaceRepairOnly: true })).rejects.toThrow("source or task owner changed");
+  });
+
   it.each(["pending_stop", "wrong_receipt", "missing_source", "unrecorded_source", "empty_sources"])("rejects repair with %s even when releasedAt is populated", async scenario => {
     const f = await seed();
     if (scenario === "pending_stop") {