fix(environments): tear down retain-on-failure sandboxes when their environment is deleted

A retain_on_failure lease keeps a failed run's sandbox stopped as a workspace-restore
source. Environment deletion counted and destroyed only reuse_by_environment leases,
so these sandboxes neither blocked the delete nor were destroyed by it: three survived
the Daytona warm wave's environment delete. Count them in the delete blast radius and
destroy them with the same consented teardown.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-08 11:38:40 -06:00
1 parent a65626a2a2
commit cc02778296
3 files changed
+32 -3

No files matched your search

@@ -5534,6 +5534,29 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
});
});
it("destroys a retained restore-source sandbox (retain_on_failure) when its environment is deleted", async () => {
const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease();
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, runId));
// A failed run's sandbox kept, stopped, as a workspace-restore source.
await db.update(environmentLeases).set({ leasePolicy: "retain_on_failure", status: "released", cleanupStatus: "success",
releasedAt: new Date() }).where(eq(environmentLeases.id, reusableLease.id));
expect(await environmentService(db).getDeleteBlastRadius(reusableLease.environmentId!))
.toMatchObject({ reusableSandboxLeaseCount: 1, canDelete: false });
const workerManager = {
isRunning: vi.fn((id: string) => id === pluginId),
call: vi.fn(async (_pluginId: string, method: string) => {
if (method === "environmentDestroyLease") return { providerLeaseId: reusableLease.providerLeaseId, state: "destroyed" };
throw new Error(`Unexpected plugin method: ${method}`);
}),
getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
} as unknown as PluginWorkerManager;
const result = await environmentRuntimeService(db, { pluginWorkerManager: workerManager })
.destroyReusableSandboxLeasesForEnvironment({ environmentId: reusableLease.environmentId!, failureReason: "environment_deleted" });
expect(result).toEqual({ destroyed: 1, failed: 0, skippedLiveRun: 0 });
await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
status: "expired", failureReason: "environment_deleted", cleanupStatus: "success" });
});
it("destroys a failed reusable sandbox after successful release before deleting its environment", async () => {
const { pluginId, companyId, runId, environment, reusableLease } = await seedReusablePluginSandboxLease();
const workerManager = {
+6 -2
View File
@@ -4321,7 +4321,10 @@ export function environmentRuntimeService(
.where(
and(
eq(environmentLeases.environmentId, input.environmentId),
eq(environmentLeases.leasePolicy, "reuse_by_environment"),
// Reusable sandboxes, and retain-on-failure sandboxes kept stopped as
// workspace-restore sources: both outlive their run, so a deleted
// environment must tear them down or they are orphaned.
inArray(environmentLeases.leasePolicy, ["reuse_by_environment", "retain_on_failure"]),
// A failed run may have stopped its reusable sandbox without
// destroying it; that provider handle still needs scoped teardown.
inArray(environmentLeases.status, ["active", "released", "retained", "failed"]),
@@ -4360,7 +4363,8 @@ export function environmentRuntimeService(
skippedLiveRun += 1;
continue;
}
const lease = await destroyClaimedSandboxLease({ environment, leaseRow, failureReason });
const lease = await destroyClaimedSandboxLease({ environment, leaseRow, failureReason, anyLeasePolicy: true,
scopeCondition: and(inArray(environmentLeases.leasePolicy, ["reuse_by_environment", "retain_on_failure"])) });
if (!lease) {
// Lost to a racing resume or a concurrent terminal transition — the
// lease is no longer ours to destroy.
+3 -1
View File
@@ -1427,7 +1427,9 @@ export function environmentService(db: Db) {
.where(
and(
eq(environmentLeases.environmentId, id),
eq(environmentLeases.leasePolicy, "reuse_by_environment"),
// A kept sandbox outlives its run: a reusable one for the next turn,
// and a retain-on-failure one as a workspace-restore source.
inArray(environmentLeases.leasePolicy, ["reuse_by_environment", "retain_on_failure"]),
inArray(environmentLeases.status, ["active", "released", "retained", "failed"]),
),
),