From cc027782967fcc837349dcd58f5c6d23cacb227b Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 8 Nov 2026 11:38:40 -0600 Subject: [PATCH] fix(environments): tear down retain-on-failure sandboxes when their environment is deleted A retain_on_failure lease keeps a failed run's sandbox stopped as a workspace-restore source. Environment deletion counted and destroyed only reuse_by_environment leases, so these sandboxes neither blocked the delete nor were destroyed by it: three survived the Daytona warm wave's environment delete. Count them in the delete blast radius and destroy them with the same consented teardown. Co-Authored-By: Claude Opus 5.5 --- .../src/__tests__/environment-runtime.test.ts | 23 +++++++++++++++++++ server/src/services/environment-runtime.ts | 8 +++++-- server/src/services/environments.ts | 4 +++- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 34c9b18aa09..4446aee2771 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -5534,6 +5534,29 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { }); }); + it("destroys a retained restore-source sandbox (retain_on_failure) when its environment is deleted", async () => { + const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease(); + await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, runId)); + // A failed run's sandbox kept, stopped, as a workspace-restore source. + await db.update(environmentLeases).set({ leasePolicy: "retain_on_failure", status: "released", cleanupStatus: "success", + releasedAt: new Date() }).where(eq(environmentLeases.id, reusableLease.id)); + expect(await environmentService(db).getDeleteBlastRadius(reusableLease.environmentId!)) + .toMatchObject({ reusableSandboxLeaseCount: 1, canDelete: false }); + const workerManager = { + isRunning: vi.fn((id: string) => id === pluginId), + call: vi.fn(async (_pluginId: string, method: string) => { + if (method === "environmentDestroyLease") return { providerLeaseId: reusableLease.providerLeaseId, state: "destroyed" }; + throw new Error(`Unexpected plugin method: ${method}`); + }), + getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), + } as unknown as PluginWorkerManager; + const result = await environmentRuntimeService(db, { pluginWorkerManager: workerManager }) + .destroyReusableSandboxLeasesForEnvironment({ environmentId: reusableLease.environmentId!, failureReason: "environment_deleted" }); + expect(result).toEqual({ destroyed: 1, failed: 0, skippedLiveRun: 0 }); + await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ + status: "expired", failureReason: "environment_deleted", cleanupStatus: "success" }); + }); + it("destroys a failed reusable sandbox after successful release before deleting its environment", async () => { const { pluginId, companyId, runId, environment, reusableLease } = await seedReusablePluginSandboxLease(); const workerManager = { diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 2a9895ece08..02d108cf25a 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -4321,7 +4321,10 @@ export function environmentRuntimeService( .where( and( eq(environmentLeases.environmentId, input.environmentId), - eq(environmentLeases.leasePolicy, "reuse_by_environment"), + // Reusable sandboxes, and retain-on-failure sandboxes kept stopped as + // workspace-restore sources: both outlive their run, so a deleted + // environment must tear them down or they are orphaned. + inArray(environmentLeases.leasePolicy, ["reuse_by_environment", "retain_on_failure"]), // A failed run may have stopped its reusable sandbox without // destroying it; that provider handle still needs scoped teardown. inArray(environmentLeases.status, ["active", "released", "retained", "failed"]), @@ -4360,7 +4363,8 @@ export function environmentRuntimeService( skippedLiveRun += 1; continue; } - const lease = await destroyClaimedSandboxLease({ environment, leaseRow, failureReason }); + const lease = await destroyClaimedSandboxLease({ environment, leaseRow, failureReason, anyLeasePolicy: true, + scopeCondition: and(inArray(environmentLeases.leasePolicy, ["reuse_by_environment", "retain_on_failure"])) }); if (!lease) { // Lost to a racing resume or a concurrent terminal transition — the // lease is no longer ours to destroy. diff --git a/server/src/services/environments.ts b/server/src/services/environments.ts index 72590988116..b7f57490c4e 100644 --- a/server/src/services/environments.ts +++ b/server/src/services/environments.ts @@ -1427,7 +1427,9 @@ export function environmentService(db: Db) { .where( and( eq(environmentLeases.environmentId, id), - eq(environmentLeases.leasePolicy, "reuse_by_environment"), + // A kept sandbox outlives its run: a reusable one for the next turn, + // and a retain-on-failure one as a workspace-restore source. + inArray(environmentLeases.leasePolicy, ["reuse_by_environment", "retain_on_failure"]), inArray(environmentLeases.status, ["active", "released", "retained", "failed"]), ), ),