feat: guard environment delete on active runtime use, make cleanup atomic

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/routes/environments.ts

@@ -12,7 +12,6 @@ import {
   redactEnvironmentCustomImageTemplate,
   startEnvironmentCustomImageSetupSessionSchema,
   type EnvironmentDeleteBlastRadius,
-  type EnvironmentCustomImageSetupSessionStatus,
   updateEnvironmentSchema,
 } from "@paperclipai/shared";
 import { conflict, forbidden, unprocessable } from "../errors.js";
@@ -20,10 +19,8 @@ import { validate } from "../middleware/validate.js";
 import { logger } from "../middleware/logger.js";
 import {
   environmentCustomImageService,
-  issueService,
   instanceSettingsService,
   logActivity,
-  projectService,
 } from "../services/index.js";
 import {
   environmentCustomImageTerminalConnectionRegistry,
@@ -39,7 +36,6 @@ import {
   collectEnvironmentSecretRefs,
   normalizeEnvironmentConfigForPersistence,
   normalizeEnvironmentConfigForProbe,
-  readSshEnvironmentPrivateKeySecretId,
   type ParsedEnvironmentConfig,
 } from "../services/environment-config.js";
 import { probeEnvironment } from "../services/environment-probe.js";
@@ -49,17 +45,10 @@ import { getConfiguredSecretProvider } from "../secrets/configured-provider.js";
 import { assertBoardOrgAccess, getActorInfo } from "./authz.js";
 import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
 import { environmentService } from "../services/environments.js";
-import { executionWorkspaceService } from "../services/execution-workspaces.js";
+import { environmentRuntimeService } from "../services/environment-runtime.js";
 
-const ACTIVE_CUSTOM_IMAGE_SETUP_STATUSES = [
-  "starting",
-  "waiting_for_user",
-  "capturing",
-] as const satisfies readonly EnvironmentCustomImageSetupSessionStatus[];
-
-function isActiveCustomImageSetupStatus(status: EnvironmentCustomImageSetupSessionStatus): boolean {
-  return (ACTIVE_CUSTOM_IMAGE_SETUP_STATUSES as readonly string[]).includes(status);
-}
+const ACTIVE_CUSTOM_IMAGE_SETUP_STATUSES_FOR_DELETE: ReadonlySet<string> =
+  new Set(["starting", "waiting_for_user", "capturing"]);
 
 export function environmentRoutes(
   db: Db,
@@ -70,10 +59,10 @@ export function environmentRoutes(
   const customImages = environmentCustomImageService(db, {
     pluginWorkerManager: options.pluginWorkerManager,
   });
-  const executionWorkspaces = executionWorkspaceService(db);
-  const issues = issueService(db);
   const instanceSettings = instanceSettingsService(db);
-  const projects = projectService(db);
+  const environmentRuntime = environmentRuntimeService(db, {
+    pluginWorkerManager: options.pluginWorkerManager,
+  });
   const secrets = secretService(db);
   const strictSecretsMode = process.env.PAPERCLIP_SECRETS_STRICT_MODE === "true";
 
@@ -290,9 +279,74 @@ export function environmentRoutes(
     if (impact.staticReferences.isInstanceDefault) {
       return "Cannot delete the current instance default environment. Set a new default environment before deleting this one.";
     }
+    if (impact.activeRuntimeUse.hasActiveRuntimeUse) {
+      return "Cannot delete the environment while active runtime leases or setup sessions are still running. Retry with force after cleanup completes.";
+    }
     return null;
   }
 
+  function parseEnvironmentDeleteForce(req: Request): boolean {
+    return Boolean(
+      req.body
+      && typeof req.body === "object"
+      && !Array.isArray(req.body)
+      && (req.body as { force?: unknown }).force === true,
+    );
+  }
+
+  function rejectEnvironmentDeleteActiveRuntimeUse(impact: EnvironmentDeleteBlastRadius): never {
+    throw conflict(
+      "Environment has active runtime leases or setup sessions. Repeat the delete request with force: true to clean up active runtime use before deletion.",
+      {
+        requiresForce: true,
+        activeRuntimeUse: impact.activeRuntimeUse,
+      },
+    );
+  }
+
+  async function cleanupActiveRuntimeUseBeforeEnvironmentDelete(input: {
+    environmentId: string;
+    impact: EnvironmentDeleteBlastRadius;
+  }) {
+    const releasedLeases = input.impact.activeRuntimeUse.activeLeaseCount > 0
+      ? await environmentRuntime.releaseActiveLeasesForEnvironment(input.environmentId, "released")
+      : [];
+    const failedLeaseCleanups = releasedLeases.filter((record) =>
+      record.lease.cleanupStatus === "failed"
+      || record.lease.status === "pending_cleanup"
+      || record.lease.status === "retained"
+    );
+    if (failedLeaseCleanups.length > 0) {
+      throw conflict(
+        "Environment delete could not clean up all active runtime leases. Retry after the failed lease cleanup is resolved.",
+        {
+          failedLeaseIds: failedLeaseCleanups.map((record) => record.lease.id),
+        },
+      );
+    }
+
+    if (input.impact.activeRuntimeUse.activeCustomImageSetupSessionCount > 0) {
+      const overview = await customImages.getOverview({ environmentId: input.environmentId });
+      if (overview.activeSession) {
+        const cancelled = await customImages.cancelSetupSession({
+          sessionId: overview.activeSession.id,
+          reason: "environment_deleted",
+        });
+        environmentCustomImageTerminalSessionStore.deleteBySetupSessionId(overview.activeSession.id);
+        environmentCustomImageTerminalConnectionRegistry.closeBySetupSessionId(
+          overview.activeSession.id,
+          "environment_deleted",
+        );
+        if (cancelled.status === "failed" || ACTIVE_CUSTOM_IMAGE_SETUP_STATUSES_FOR_DELETE.has(cancelled.status)) {
+          throw conflict(
+            "Environment delete could not cancel the active custom image setup session. Retry after the failed setup cleanup is resolved.",
+            { setupSessionId: cancelled.id },
+          );
+        }
+      }
+    }
+  }
+
   function rejectEnvironmentDelete(input: {
     actor: ReturnType<typeof getActorInfo>;
     environment: { id: string; driver: string };
@@ -824,38 +878,22 @@ export function environmentRoutes(
       res.status(404).json({ error: "Environment not found" });
       return;
     }
-
-    const hasStaticBlocker = impact.deleteBlockedReasons.some(
-      (r) => r === "managed_local" || r === "instance_default",
-    );
-    if (hasStaticBlocker) {
+    const forceDelete = parseEnvironmentDeleteForce(req);
+    const staticDeleteBlockedReasons = impact.deleteBlockedReasons.filter((reason) => reason !== "active_runtime_use");
+    if (staticDeleteBlockedReasons.length > 0) {
       rejectEnvironmentDelete({ actor, environment: existing, impact });
     }
-
-    const hasRuntimeBlocker = impact.deleteBlockedReasons.includes("active_runtime_use");
-    const forceDelete = req.query.force === "true";
-    if (hasRuntimeBlocker && !forceDelete) {
-      rejectEnvironmentDelete({ actor, environment: existing, impact });
-    }
-
-    if (hasRuntimeBlocker && forceDelete) {
-      const overview = await customImages.getOverview({ environmentId: existing.id });
-      if (overview.activeSession) {
-        const cancelled = await customImages.cancelSetupSession({
-          sessionId: overview.activeSession.id,
-          reason: "environment_force_delete",
-        });
-        if (isActiveCustomImageSetupStatus(cancelled.status)) {
-          throw conflict(
-            "Active custom image setup session could not be cancelled. Retry or wait for setup to complete before deleting.",
-            { deleteBlockedReasons: ["active_runtime_use"] },
-          );
-        }
+    if (impact.activeRuntimeUse.hasActiveRuntimeUse) {
+      if (!forceDelete) {
+        rejectEnvironmentDeleteActiveRuntimeUse(impact);
       }
-      await svc.releaseActiveLeasesForEnvironment(existing.id);
+      await cleanupActiveRuntimeUseBeforeEnvironmentDelete({
+        environmentId: existing.id,
+        impact,
+      });
     }
 
-    const removed = await svc.removeIfDeletable(existing.id);
+    const removed = await svc.removeIfDeletableWithCleanup(existing.id);
     if (!removed) {
       const latestImpact = await svc.getDeleteBlastRadius(existing.id);
       if (!latestImpact) {
@@ -864,29 +902,6 @@ export function environmentRoutes(
       }
       rejectEnvironmentDelete({ actor, environment: existing, impact: latestImpact });
     }
-    const companyIds = await instanceSettings.listCompanyIds();
-    await Promise.all(
-      companyIds.flatMap((companyId) => [
-        executionWorkspaces.clearEnvironmentSelection(companyId, existing.id),
-        issues.clearExecutionWorkspaceEnvironmentSelection(companyId, existing.id),
-        projects.clearExecutionWorkspaceEnvironmentSelection(companyId, existing.id),
-        secrets.syncEnvBindingsForTarget(
-          companyId,
-          { targetType: "environment", targetId: existing.id },
-          {},
-        ),
-        secrets.syncSecretRefsForTarget(
-          companyId,
-          { targetType: "environment", targetId: existing.id },
-          [],
-          { replaceAll: true },
-        ),
-      ]),
-    );
-    const secretId = readSshEnvironmentPrivateKeySecretId(existing);
-    if (secretId) {
-      await secrets.remove(secretId);
-    }
     await logInstanceEnvironmentActivity({
       actor,
       action: "environment.deleted",

server/src/routes/openapi.ts

@@ -3747,7 +3747,10 @@ registry.registerPath({
   path: "/api/environments/{id}",
   tags: ["environments"],
   summary: "Delete an environment",
-  request: { params: z.object({ id: z.string() }) },
+  request: {
+    params: z.object({ id: z.string() }),
+    body: jsonBody(z.object({ force: z.boolean().optional() }).strict()),
+  },
   responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden, 404: r.notFound, 409: r.conflict },
 });
 

server/src/services/environment-runtime.ts

@@ -1803,6 +1803,69 @@ export function environmentRuntimeService(
       return released;
     },
 
+    async releaseActiveLeasesForEnvironment(
+      environmentId: string,
+      status: Extract<EnvironmentLeaseStatus, "released" | "expired" | "failed"> = "released",
+    ): Promise<EnvironmentRuntimeLeaseRecord[]> {
+      const environment = await environmentsSvc.getById(environmentId);
+      if (!environment) return [];
+      const leaseRows = await db
+        .select()
+        .from(environmentLeases)
+        .where(
+          and(
+            eq(environmentLeases.environmentId, environmentId),
+            inArray(environmentLeases.status, ["active"]),
+          ),
+        );
+      if (leaseRows.length === 0) return [];
+
+      const released: EnvironmentRuntimeLeaseRecord[] = [];
+      for (const leaseRow of leaseRows) {
+        const leaseSnapshot = toEnvironmentLeaseSnapshot(leaseRow);
+        const driver = getDriver(getLeaseDriverKey(leaseSnapshot, environment));
+        const lease = await (async () => {
+          if (!driver) {
+            return environmentsSvc.releaseLease(leaseSnapshot.id, "pending_cleanup", {
+              failureReason: "environment_delete_missing_runtime_driver",
+              cleanupStatus: "failed",
+            });
+          }
+          if (leaseSnapshot.leasePolicy === "reuse_by_environment") {
+            if (!driver.destroyRunLease) {
+              return environmentsSvc.releaseLease(leaseSnapshot.id, "pending_cleanup", {
+                failureReason: "environment_delete_missing_reusable_lease_destroyer",
+                cleanupStatus: "failed",
+              });
+            }
+            return driver.destroyRunLease({
+              environment,
+              lease: leaseSnapshot,
+              failureReason: "environment_deleted",
+            });
+          }
+          return driver.releaseRunLease({
+            environment,
+            lease: leaseSnapshot,
+            status,
+          });
+        })();
+        if (!lease) continue;
+
+        released.push({
+          environment,
+          lease,
+          leaseContext: {
+            executionWorkspaceId: lease.executionWorkspaceId,
+            executionWorkspaceMode:
+              (lease.metadata?.executionWorkspaceMode as ExecutionWorkspace["mode"] | null | undefined) ?? null,
+          },
+        });
+      }
+
+      return released;
+    },
+
     async destroyReusableSandboxLeases(input: {
       companyId: string;
       issueId?: string | null;

server/src/services/environments.ts

@@ -1,7 +1,8 @@
-import { and, desc, eq, inArray, ne, sql } from "drizzle-orm";
+import { and, desc, eq, inArray, like, ne, or, sql } from "drizzle-orm";
 import type { Db } from "@paperclipai/db";
 import {
   agents,
+  companySecrets,
   companySecretBindings,
   environmentCustomImageSetupSessions,
   environmentLeases,
@@ -10,6 +11,7 @@ import {
   instanceSettings,
   issues,
   projects,
+  userSecretDeclarations,
 } from "@paperclipai/db";
 import {
   ENVIRONMENT_DRIVERS,
@@ -180,6 +182,15 @@ function countFromRows(rows: Array<{ count: number | string | null | undefined }
   return Number(rows[0]?.count ?? 0);
 }
 
+function readSshPrivateKeySecretIdFromEnvironment(row: EnvironmentRow): string | null {
+  if (row.driver !== "ssh") return null;
+  const config = cloneRecord(row.config, {}) ?? {};
+  const ref = config.privateKeySecretRef;
+  if (!ref || typeof ref !== "object" || Array.isArray(ref)) return null;
+  const secretId = (ref as Record<string, unknown>).secretId;
+  return typeof secretId === "string" && secretId.length > 0 ? secretId : null;
+}
+
 export function environmentService(db: Db) {
   return {
     list: async (
@@ -480,6 +491,92 @@ export function environmentService(db: Db) {
       return row ? toEnvironment(row) : null;
     },
 
+    removeIfDeletableWithCleanup: async (id: string): Promise<Environment | null> => {
+      const now = new Date();
+      return db.transaction(async (tx) => {
+        const row = await tx
+          .delete(environments)
+          .where(
+            and(
+              eq(environments.id, id),
+              ne(environments.driver, "local"),
+              sql`not exists (
+                select 1 from ${instanceSettings}
+                where ${instanceSettings.defaultEnvironmentId} = ${environments.id}
+              )`,
+              sql`not exists (
+                select 1 from ${environmentLeases}
+                where ${environmentLeases.environmentId} = ${environments.id}
+                  and ${environmentLeases.status} = 'active'
+              )`,
+              sql`not exists (
+                select 1 from ${environmentCustomImageSetupSessions}
+                where ${environmentCustomImageSetupSessions.environmentId} = ${environments.id}
+                  and ${environmentCustomImageSetupSessions.status} in ('starting', 'waiting_for_user', 'capturing')
+              )`,
+            ),
+          )
+          .returning()
+          .then((rows) => rows[0] ?? null);
+        if (!row) return null;
+
+        await tx
+          .update(executionWorkspaces)
+          .set({
+            metadata: sql`jsonb_set(coalesce(${executionWorkspaces.metadata}, '{}'::jsonb), '{config,environmentId}', 'null'::jsonb, true)`,
+            updatedAt: now,
+          })
+          .where(sql`${executionWorkspaces.metadata} -> 'config' ->> 'environmentId' = ${id}`);
+
+        await tx
+          .update(issues)
+          .set({
+            executionWorkspaceSettings:
+              sql`jsonb_set(coalesce(${issues.executionWorkspaceSettings}, '{}'::jsonb), '{environmentId}', 'null'::jsonb, true)`,
+            updatedAt: now,
+          })
+          .where(sql`${issues.executionWorkspaceSettings} ->> 'environmentId' = ${id}`);
+
+        await tx
+          .update(projects)
+          .set({
+            executionWorkspacePolicy:
+              sql`jsonb_set(coalesce(${projects.executionWorkspacePolicy}, '{}'::jsonb), '{environmentId}', 'null'::jsonb, true)`,
+            updatedAt: now,
+          })
+          .where(sql`${projects.executionWorkspacePolicy} ->> 'environmentId' = ${id}`);
+
+        await tx
+          .delete(userSecretDeclarations)
+          .where(
+            and(
+              eq(userSecretDeclarations.targetType, "environment"),
+              eq(userSecretDeclarations.targetId, id),
+              or(
+                eq(userSecretDeclarations.configPath, "env"),
+                like(userSecretDeclarations.configPath, "env.%"),
+              ),
+            ),
+          );
+
+        await tx
+          .delete(companySecretBindings)
+          .where(
+            and(
+              eq(companySecretBindings.targetType, "environment"),
+              eq(companySecretBindings.targetId, id),
+            ),
+          );
+
+        const privateKeySecretId = readSshPrivateKeySecretIdFromEnvironment(row);
+        if (privateKeySecretId) {
+          await tx.delete(companySecrets).where(eq(companySecrets.id, privateKeySecretId));
+        }
+
+        return toEnvironment(row);
+      });
+    },
+
     getDeleteBlastRadius: async (id: string): Promise<EnvironmentDeleteBlastRadius | null> => {
       const environment = await db
         .select({
@@ -552,13 +649,16 @@ export function environmentService(db: Db) {
 
       const isManagedLocal = environment.driver === "local";
       const isInstanceDefault = countFromRows(instanceDefaultRows) > 0;
-      const activeLeaseCount = countFromRows(activeLeaseRows);
-      const activeCustomImageSetupSessionCount = countFromRows(activeSetupRows);
-      const hasActiveRuntimeUse = activeLeaseCount > 0 || activeCustomImageSetupSessionCount > 0;
       const deleteBlockedReasons: EnvironmentDeleteBlockedReason[] = [];
       if (isManagedLocal) deleteBlockedReasons.push("managed_local");
       if (isInstanceDefault) deleteBlockedReasons.push("instance_default");
-      if (hasActiveRuntimeUse) deleteBlockedReasons.push("active_runtime_use");
+      const activeLeaseCount = countFromRows(activeLeaseRows);
+      const activeCustomImageSetupSessionCount = countFromRows(activeSetupRows);
+      const hasActiveRuntimeUse =
+        activeLeaseCount > 0 || activeCustomImageSetupSessionCount > 0;
+      if (hasActiveRuntimeUse) {
+        deleteBlockedReasons.push("active_runtime_use");
+      }
 
       return {
         environmentId: id,
@@ -581,16 +681,6 @@ export function environmentService(db: Db) {
       };
     },
 
-    releaseActiveLeasesForEnvironment: async (id: string): Promise<number> => {
-      const now = new Date();
-      const rows = await db
-        .update(environmentLeases)
-        .set({ status: "released", releasedAt: now, lastUsedAt: now, updatedAt: now })
-        .where(and(eq(environmentLeases.environmentId, id), eq(environmentLeases.status, "active")))
-        .returning();
-      return rows.length;
-    },
-
     listLeases: async (
       environmentId: string,
       filters: {