fix(server): admit verified warm lease retention

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts

@@ -366,7 +366,8 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
     }
   });
 
-  it.each(["active_lease", "pending_cleanup", "failed_cleanup", "finalizer_lease", "workspace_finalization"])(
+  it.each(["active_lease", "pending_cleanup", "failed_cleanup", "finalizer_lease", "workspace_finalization",
+    "retained_ready", "retained_missing_receipt", "retained_failed", "retained_wrong_policy"])(
     "waits for durable cleanup from another controller: %s", async (pending) => {
       const { companyId, agentId } = await seedCompanyAndAgent();
       const issueId = randomUUID(), previousId = randomUUID();
@@ -376,30 +377,36 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
         invocationSource: "assignment", contextSnapshot: { issueId }, finishedAt: new Date() });
       await db.update(issues).set({ executionRunId: previousId }).where(eq(issues.id, issueId));
       const leaseId = randomUUID();
-      const hasLease = ["active_lease", "pending_cleanup", "failed_cleanup"].includes(pending);
+      const retained = pending.startsWith("retained_");
+      const hasLease = retained || ["active_lease", "pending_cleanup", "failed_cleanup"].includes(pending);
       if (hasLease) await db.insert(environmentLeases).values({ id: leaseId, companyId, issueId,
         heartbeatRunId: previousId, provider: "daytona",
-        status: pending === "pending_cleanup" ? "pending_cleanup" : "released",
-        releasedAt: pending === "active_lease" ? null : new Date(),
-        cleanupStatus: pending === "failed_cleanup" ? "failed" : null });
+        status: retained ? "retained" : pending === "pending_cleanup" ? "pending_cleanup" : "released",
+        leasePolicy: pending === "retained_wrong_policy" ? "retain_on_failure" : retained ? "reuse_by_environment" : "ephemeral",
+        releasedAt: retained || pending === "active_lease" ? null : new Date(),
+        cleanupStatus: ["retained_ready", "retained_wrong_policy"].includes(pending) ? "success"
+          : ["failed_cleanup", "retained_failed"].includes(pending) ? "failed" : null });
       await db.insert(nativeRunFinalizations).values({ runId: previousId, companyId, issueId,
         phase: pending === "workspace_finalization" ? "workspace_finalizing" : "committed",
         leaseOwner: pending === "finalizer_lease" ? "another-controller" : null });
       const next = await seedQueuedRun({ companyId, agentId, issueId, wakeReason: "issue_assigned", invocationSource: "assignment" });
-      // No in-memory executor exists in this service. Durable ownership alone
-      // must fence a second controller until the prior cleanup settles.
-      await heartbeat.resumeQueuedRuns();
-      expect((await heartbeat.getRun(next.runId))?.status).toBe("queued");
-      expect(mockAdapterExecute).not.toHaveBeenCalled();
-      expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0].executionRunId).toBe(previousId);
-      if (hasLease) await db.update(environmentLeases).set({ status: "released", releasedAt: new Date(), cleanupStatus: "succeeded" }).where(eq(environmentLeases.id, leaseId));
-      await db.update(nativeRunFinalizations).set({ phase: "committed", leaseOwner: null }).where(eq(nativeRunFinalizations.runId, previousId));
       mockAdapterExecute.mockImplementation(async () => {
         await db.update(issues).set({ status: "done" }).where(eq(issues.id, issueId));
         return { exitCode: 0, signal: null, timedOut: false, errorMessage: null,
           summary: "Task completed", provider: "test", model: "test-model" };
       });
+      // No in-memory executor exists in this service. Durable ownership alone
+      // must fence a second controller until cleanup settles. A verified warm
+      // retention receipt is already a settled boundary and allows continuation.
       await heartbeat.resumeQueuedRuns();
+      if (pending !== "retained_ready") {
+        expect((await heartbeat.getRun(next.runId))?.status).toBe("queued");
+        expect(mockAdapterExecute).not.toHaveBeenCalled();
+        expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0].executionRunId).toBe(previousId);
+        if (hasLease) await db.update(environmentLeases).set({ status: "released", releasedAt: new Date(), cleanupStatus: "success" }).where(eq(environmentLeases.id, leaseId));
+        await db.update(nativeRunFinalizations).set({ phase: "committed", leaseOwner: null }).where(eq(nativeRunFinalizations.runId, previousId));
+        await heartbeat.resumeQueuedRuns();
+      }
       await heartbeat.drainActiveRunExecutions();
       expect((await heartbeat.getRun(next.runId))?.status).toBe("succeeded");
       expect(mockAdapterExecute).toHaveBeenCalledTimes(1);

server/src/services/heartbeat.ts

@@ -17328,7 +17328,12 @@ export function heartbeatService(
           .from(environmentLeases).where(and(
             eq(environmentLeases.companyId, run.companyId),
             eq(environmentLeases.heartbeatRunId, owner.executionRunId),
-            or(isNull(environmentLeases.releasedAt),
+            or(and(isNull(environmentLeases.releasedAt),
+                // Warm release deliberately retains the sandbox. Its successful
+                // receipt settles the old run without destroying the resource.
+                sql`not coalesce(${environmentLeases.status} = 'retained'
+                  and ${environmentLeases.leasePolicy} = 'reuse_by_environment'
+                  and ${environmentLeases.cleanupStatus} = 'success', false)`),
               eq(environmentLeases.status, "pending_cleanup"),
               eq(environmentLeases.cleanupStatus, "failed")),
           )).limit(1);