From e5bce4681768824201a377598ea532ff39ebf46e Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 24 Sep 2026 17:38:47 -0700 Subject: [PATCH] fix(server): admit verified warm lease retention Co-Authored-By: Paperclip --- ...heartbeat-stale-queue-invalidation.test.ts | 33 +++++++++++-------- server/src/services/heartbeat.ts | 7 +++- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts index 087f27bd76a..882a81e5d2b 100644 --- a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts +++ b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts @@ -366,7 +366,8 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { } }); - it.each(["active_lease", "pending_cleanup", "failed_cleanup", "finalizer_lease", "workspace_finalization"])( + it.each(["active_lease", "pending_cleanup", "failed_cleanup", "finalizer_lease", "workspace_finalization", + "retained_ready", "retained_missing_receipt", "retained_failed", "retained_wrong_policy"])( "waits for durable cleanup from another controller: %s", async (pending) => { const { companyId, agentId } = await seedCompanyAndAgent(); const issueId = randomUUID(), previousId = randomUUID(); @@ -376,30 +377,36 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { invocationSource: "assignment", contextSnapshot: { issueId }, finishedAt: new Date() }); await db.update(issues).set({ executionRunId: previousId }).where(eq(issues.id, issueId)); const leaseId = randomUUID(); - const hasLease = ["active_lease", "pending_cleanup", "failed_cleanup"].includes(pending); + const retained = pending.startsWith("retained_"); + const hasLease = retained || ["active_lease", "pending_cleanup", "failed_cleanup"].includes(pending); if (hasLease) await db.insert(environmentLeases).values({ id: leaseId, companyId, issueId, heartbeatRunId: previousId, provider: "daytona", - status: pending === "pending_cleanup" ? "pending_cleanup" : "released", - releasedAt: pending === "active_lease" ? null : new Date(), - cleanupStatus: pending === "failed_cleanup" ? "failed" : null }); + status: retained ? "retained" : pending === "pending_cleanup" ? "pending_cleanup" : "released", + leasePolicy: pending === "retained_wrong_policy" ? "retain_on_failure" : retained ? "reuse_by_environment" : "ephemeral", + releasedAt: retained || pending === "active_lease" ? null : new Date(), + cleanupStatus: ["retained_ready", "retained_wrong_policy"].includes(pending) ? "success" + : ["failed_cleanup", "retained_failed"].includes(pending) ? "failed" : null }); await db.insert(nativeRunFinalizations).values({ runId: previousId, companyId, issueId, phase: pending === "workspace_finalization" ? "workspace_finalizing" : "committed", leaseOwner: pending === "finalizer_lease" ? "another-controller" : null }); const next = await seedQueuedRun({ companyId, agentId, issueId, wakeReason: "issue_assigned", invocationSource: "assignment" }); - // No in-memory executor exists in this service. Durable ownership alone - // must fence a second controller until the prior cleanup settles. - await heartbeat.resumeQueuedRuns(); - expect((await heartbeat.getRun(next.runId))?.status).toBe("queued"); - expect(mockAdapterExecute).not.toHaveBeenCalled(); - expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0].executionRunId).toBe(previousId); - if (hasLease) await db.update(environmentLeases).set({ status: "released", releasedAt: new Date(), cleanupStatus: "succeeded" }).where(eq(environmentLeases.id, leaseId)); - await db.update(nativeRunFinalizations).set({ phase: "committed", leaseOwner: null }).where(eq(nativeRunFinalizations.runId, previousId)); mockAdapterExecute.mockImplementation(async () => { await db.update(issues).set({ status: "done" }).where(eq(issues.id, issueId)); return { exitCode: 0, signal: null, timedOut: false, errorMessage: null, summary: "Task completed", provider: "test", model: "test-model" }; }); + // No in-memory executor exists in this service. Durable ownership alone + // must fence a second controller until cleanup settles. A verified warm + // retention receipt is already a settled boundary and allows continuation. await heartbeat.resumeQueuedRuns(); + if (pending !== "retained_ready") { + expect((await heartbeat.getRun(next.runId))?.status).toBe("queued"); + expect(mockAdapterExecute).not.toHaveBeenCalled(); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0].executionRunId).toBe(previousId); + if (hasLease) await db.update(environmentLeases).set({ status: "released", releasedAt: new Date(), cleanupStatus: "success" }).where(eq(environmentLeases.id, leaseId)); + await db.update(nativeRunFinalizations).set({ phase: "committed", leaseOwner: null }).where(eq(nativeRunFinalizations.runId, previousId)); + await heartbeat.resumeQueuedRuns(); + } await heartbeat.drainActiveRunExecutions(); expect((await heartbeat.getRun(next.runId))?.status).toBe("succeeded"); expect(mockAdapterExecute).toHaveBeenCalledTimes(1); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 1978d368457..abf7e8a3425 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -17328,7 +17328,12 @@ export function heartbeatService( .from(environmentLeases).where(and( eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, owner.executionRunId), - or(isNull(environmentLeases.releasedAt), + or(and(isNull(environmentLeases.releasedAt), + // Warm release deliberately retains the sandbox. Its successful + // receipt settles the old run without destroying the resource. + sql`not coalesce(${environmentLeases.status} = 'retained' + and ${environmentLeases.leasePolicy} = 'reuse_by_environment' + and ${environmentLeases.cleanupStatus} = 'success', false)`), eq(environmentLeases.status, "pending_cleanup"), eq(environmentLeases.cleanupStatus, "failed")), )).limit(1);