fix: preserve workspace finalization during watchdog recovery (#16247)

Keep task completion separate from runtime and workspace settlement. Permit verified board file-only repair of successful legacy retained-source runs without authorizing replay.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/modules/active-run-watchdog/adapters/postgres.ts

@@ -6,7 +6,9 @@ import {
   activityLog,
   heartbeatRunWatchdogDecisions,
   heartbeatRuns,
+  environmentLeases,
   issues,
+  workspaceOperations,
 } from "@paperclipai/db";
 import { parseObject } from "../../../adapters/utils.js";
 import { visibleIssueCondition } from "../../../services/issue-visibility.js";
@@ -65,6 +67,27 @@ function toRunSnapshot(row: typeof heartbeatRuns.$inferSelect): RunSnapshot {
   };
 }
 
+function runtimeSettlementPending(companyId: string, runId: string) {
+  return sql<boolean>`(
+    ${heartbeatRuns.runtimeMode} = 'native'
+    or coalesce(${heartbeatRuns.resultJson}->>'workspaceRestoreFailure', '') <> ''
+    or exists (
+      select 1 from ${environmentLeases}
+      where ${environmentLeases.companyId} = ${companyId}
+        and ${environmentLeases.heartbeatRunId} = ${runId}
+        and ${environmentLeases.providerLeaseId} is not null
+        and ${environmentLeases.status} in ('active', 'acquiring')
+    )
+    or exists (
+      select 1 from ${workspaceOperations}
+      where ${workspaceOperations.companyId} = ${companyId}
+        and ${workspaceOperations.heartbeatRunId} = ${runId}
+        and ${workspaceOperations.phase} = 'workspace_finalize'
+        and ${workspaceOperations.status} in ('running', 'failed')
+    )
+  )`;
+}
+
 export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & WatchdogWriter {
   const issuesSvc = issueService(db);
   const recoveryActionsSvc = issueRecoveryActionService(db);
@@ -119,6 +142,13 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch
     return row ? toRunSnapshot(row) : null;
   }
 
+  async function isRuntimeSettlementPending(companyId: string, runId: string): Promise<boolean> {
+    const [row] = await db.select({
+      pending: sql<boolean>`${heartbeatRuns.status} <> 'running' or ${runtimeSettlementPending(companyId, runId)}`,
+    }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId)));
+    return row?.pending ?? true;
+  }
+
   async function findLatestDecision(companyId: string, runId: string, now: Date) {
     const [quietUntilRows, dismissedRows] = await Promise.all([
       db
@@ -321,6 +351,7 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch
           eq(heartbeatRuns.id, input.run.id),
           eq(heartbeatRuns.companyId, companyId),
           eq(heartbeatRuns.status, "running"),
+          sql`not ${runtimeSettlementPending(companyId, input.run.id)}`,
         ))
         .returning();
       if (!updatedRun) return null;
@@ -477,6 +508,7 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch
   return {
     findCandidateSilentRuns,
     findRunForCompany,
+    isRuntimeSettlementPending,
     findLatestDecision,
     findOpenStaleRunEvaluation,
     findEvaluationIssueById,

server/src/modules/active-run-watchdog/application/ports.ts

@@ -53,6 +53,7 @@ export type RunningAgentSnapshot = {
 export interface WatchdogRunReader {
   findCandidateSilentRuns(input: FindCandidateSilentRunsInput): Promise<RunSnapshot[]>;
   findRunForCompany(companyId: string, runId: string): Promise<RunSnapshot | null>;
+  isRuntimeSettlementPending(companyId: string, runId: string): Promise<boolean>;
   findLatestDecision(companyId: string, runId: string, now: Date): Promise<FindLatestDecisionResult>;
   findOpenStaleRunEvaluation(companyId: string, runId: string): Promise<EvaluationIssueSnapshot | null>;
   findLatestSameRunTerminalEvidence(

server/src/modules/active-run-watchdog/application/use-cases.test.ts

@@ -38,6 +38,7 @@ describe("application", () => {
     return {
       findCandidateSilentRuns: async () => [],
       findRunForCompany: async () => null,
+      isRuntimeSettlementPending: async () => false,
       findLatestDecision: async () => ({ quietUntilDecision: null, dismissedFalsePositive: false }),
       findOpenStaleRunEvaluation: async () => null,
       findLatestSameRunTerminalEvidence: async () => null,
@@ -134,6 +135,27 @@ describe("application", () => {
   };
 
   describe("createFoldSourceResolvedRun", () => {
+    it("does not terminate a process or settle the run while its runtime owns finalization", async () => {
+      const calls: string[] = [];
+      const fold = createFoldSourceResolvedRun({
+        reader: makeReader({ isRuntimeSettlementPending: async (companyId, runId) => {
+          expect([companyId, runId]).toEqual(["company-1", "run-1"]);
+          return true;
+        } }),
+        processController: { cleanupRunProcess: async () => {
+          calls.push("cleanup");
+          return { attempted: false, outcome: "no_process_metadata", adapterType: "grok_local" };
+        } },
+        writer: { recordDecision: async () => { throw new Error("unexpected decision"); },
+          foldSourceResolvedRun: async () => { calls.push("fold"); return { kind: "folded", evaluationIssueId: null }; } },
+      });
+      expect(await fold({ run: makeRun({ processPid: 4242 }), runningAgentAdapterType: "grok_local",
+        sourceIssue: foldSourceIssue, evidence: foldEvidence, existingEvaluation: null,
+        silenceStartedAt: null, silenceAgeMs: null, now: new Date("2026-01-01T05:00:00Z") }))
+        .toEqual({ kind: "stale" });
+      expect(calls).toEqual([]);
+    });
+
     it("cleans up the process before the folding persistence call", async () => {
       const calls: string[] = [];
       const processController: RunProcessController = {
@@ -154,7 +176,7 @@ describe("application", () => {
           return { kind: "folded", evaluationIssueId: null };
         },
       };
-      const foldSourceResolvedRun = createFoldSourceResolvedRun({ writer, processController });
+      const foldSourceResolvedRun = createFoldSourceResolvedRun({ reader: makeReader(), writer, processController });
 
       const outcome = await foldSourceResolvedRun({
         run: makeRun({ processPid: 4242 }),
@@ -187,7 +209,7 @@ describe("application", () => {
         },
         foldSourceResolvedRun: async () => ({ kind: "stale" }),
       };
-      const foldSourceResolvedRun = createFoldSourceResolvedRun({ writer, processController });
+      const foldSourceResolvedRun = createFoldSourceResolvedRun({ reader: makeReader(), writer, processController });
 
       const outcome = await foldSourceResolvedRun({
         run: makeRun({ processPid: 4242 }),
@@ -396,7 +418,7 @@ describe("application", () => {
       foldSourceResolvedRun: async () => ({ kind: "folded", evaluationIssueId: null }),
       ...writerOverrides,
     };
-    return createFoldSourceResolvedRun({ writer, processController });
+    return createFoldSourceResolvedRun({ reader: makeReader(), writer, processController });
   }
 
   describe("createScanSilentActiveRuns", () => {

server/src/modules/active-run-watchdog/application/use-cases.ts

@@ -72,6 +72,7 @@ export function createBuildRunOutputSilence(deps: BuildRunOutputSilenceDeps) {
 }
 
 export type FoldSourceResolvedRunDeps = {
+  reader: Pick<WatchdogRunReader, "isRuntimeSettlementPending">;
   writer: WatchdogWriter;
   processController: RunProcessController;
 };
@@ -89,6 +90,9 @@ export type FoldSourceResolvedRunUseCaseInput = {
 
 export function createFoldSourceResolvedRun(deps: FoldSourceResolvedRunDeps) {
   return async function foldSourceResolvedRun(input: FoldSourceResolvedRunUseCaseInput): Promise<FoldOutcome> {
+    if (await deps.reader.isRuntimeSettlementPending(input.run.companyId, input.run.id)) {
+      return { kind: "stale" };
+    }
     const cleanup = await deps.processController.cleanupRunProcess({
       runId: input.run.id,
       adapterType: input.runningAgentAdapterType,

server/src/modules/active-run-watchdog/index.ts

@@ -24,6 +24,7 @@ export function createActiveRunWatchdog(db: Db, config: ActiveRunWatchdogConfig)
   const processController = createProcessAdapter();
 
   const foldSourceResolvedRun = createFoldSourceResolvedRun({
+    reader: postgresAdapter,
     writer: postgresAdapter,
     processController,
   });