From 750ad9dcc67d3e800d8d53b48db1fea5cfc99c24 Mon Sep 17 00:00:00 2001 From: Priya Raman Date: Thu, 9 Jul 2026 06:11:35 +0000 Subject: [PATCH] feat: guard environment delete on active runtime use, make cleanup atomic Co-Authored-By: Paperclip --- server/src/routes/environments.ts | 149 ++++++++++++--------- server/src/routes/openapi.ts | 5 +- server/src/services/environment-runtime.ts | 63 +++++++++ server/src/services/environments.ts | 120 ++++++++++++++--- 4 files changed, 254 insertions(+), 83 deletions(-) diff --git a/server/src/routes/environments.ts b/server/src/routes/environments.ts index 174851c48ec..11521abb746 100644 --- a/server/src/routes/environments.ts +++ b/server/src/routes/environments.ts @@ -12,7 +12,6 @@ import { redactEnvironmentCustomImageTemplate, startEnvironmentCustomImageSetupSessionSchema, type EnvironmentDeleteBlastRadius, - type EnvironmentCustomImageSetupSessionStatus, updateEnvironmentSchema, } from "@paperclipai/shared"; import { conflict, forbidden, unprocessable } from "../errors.js"; @@ -20,10 +19,8 @@ import { validate } from "../middleware/validate.js"; import { logger } from "../middleware/logger.js"; import { environmentCustomImageService, - issueService, instanceSettingsService, logActivity, - projectService, } from "../services/index.js"; import { environmentCustomImageTerminalConnectionRegistry, @@ -39,7 +36,6 @@ import { collectEnvironmentSecretRefs, normalizeEnvironmentConfigForPersistence, normalizeEnvironmentConfigForProbe, - readSshEnvironmentPrivateKeySecretId, type ParsedEnvironmentConfig, } from "../services/environment-config.js"; import { probeEnvironment } from "../services/environment-probe.js"; @@ -49,17 +45,10 @@ import { getConfiguredSecretProvider } from "../secrets/configured-provider.js"; import { assertBoardOrgAccess, getActorInfo } from "./authz.js"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; import { environmentService } from "../services/environments.js"; -import { executionWorkspaceService } from "../services/execution-workspaces.js"; +import { environmentRuntimeService } from "../services/environment-runtime.js"; -const ACTIVE_CUSTOM_IMAGE_SETUP_STATUSES = [ - "starting", - "waiting_for_user", - "capturing", -] as const satisfies readonly EnvironmentCustomImageSetupSessionStatus[]; - -function isActiveCustomImageSetupStatus(status: EnvironmentCustomImageSetupSessionStatus): boolean { - return (ACTIVE_CUSTOM_IMAGE_SETUP_STATUSES as readonly string[]).includes(status); -} +const ACTIVE_CUSTOM_IMAGE_SETUP_STATUSES_FOR_DELETE: ReadonlySet = + new Set(["starting", "waiting_for_user", "capturing"]); export function environmentRoutes( db: Db, @@ -70,10 +59,10 @@ export function environmentRoutes( const customImages = environmentCustomImageService(db, { pluginWorkerManager: options.pluginWorkerManager, }); - const executionWorkspaces = executionWorkspaceService(db); - const issues = issueService(db); const instanceSettings = instanceSettingsService(db); - const projects = projectService(db); + const environmentRuntime = environmentRuntimeService(db, { + pluginWorkerManager: options.pluginWorkerManager, + }); const secrets = secretService(db); const strictSecretsMode = process.env.PAPERCLIP_SECRETS_STRICT_MODE === "true"; @@ -290,9 +279,74 @@ export function environmentRoutes( if (impact.staticReferences.isInstanceDefault) { return "Cannot delete the current instance default environment. Set a new default environment before deleting this one."; } + if (impact.activeRuntimeUse.hasActiveRuntimeUse) { + return "Cannot delete the environment while active runtime leases or setup sessions are still running. Retry with force after cleanup completes."; + } return null; } + function parseEnvironmentDeleteForce(req: Request): boolean { + return Boolean( + req.body + && typeof req.body === "object" + && !Array.isArray(req.body) + && (req.body as { force?: unknown }).force === true, + ); + } + + function rejectEnvironmentDeleteActiveRuntimeUse(impact: EnvironmentDeleteBlastRadius): never { + throw conflict( + "Environment has active runtime leases or setup sessions. Repeat the delete request with force: true to clean up active runtime use before deletion.", + { + requiresForce: true, + activeRuntimeUse: impact.activeRuntimeUse, + }, + ); + } + + async function cleanupActiveRuntimeUseBeforeEnvironmentDelete(input: { + environmentId: string; + impact: EnvironmentDeleteBlastRadius; + }) { + const releasedLeases = input.impact.activeRuntimeUse.activeLeaseCount > 0 + ? await environmentRuntime.releaseActiveLeasesForEnvironment(input.environmentId, "released") + : []; + const failedLeaseCleanups = releasedLeases.filter((record) => + record.lease.cleanupStatus === "failed" + || record.lease.status === "pending_cleanup" + || record.lease.status === "retained" + ); + if (failedLeaseCleanups.length > 0) { + throw conflict( + "Environment delete could not clean up all active runtime leases. Retry after the failed lease cleanup is resolved.", + { + failedLeaseIds: failedLeaseCleanups.map((record) => record.lease.id), + }, + ); + } + + if (input.impact.activeRuntimeUse.activeCustomImageSetupSessionCount > 0) { + const overview = await customImages.getOverview({ environmentId: input.environmentId }); + if (overview.activeSession) { + const cancelled = await customImages.cancelSetupSession({ + sessionId: overview.activeSession.id, + reason: "environment_deleted", + }); + environmentCustomImageTerminalSessionStore.deleteBySetupSessionId(overview.activeSession.id); + environmentCustomImageTerminalConnectionRegistry.closeBySetupSessionId( + overview.activeSession.id, + "environment_deleted", + ); + if (cancelled.status === "failed" || ACTIVE_CUSTOM_IMAGE_SETUP_STATUSES_FOR_DELETE.has(cancelled.status)) { + throw conflict( + "Environment delete could not cancel the active custom image setup session. Retry after the failed setup cleanup is resolved.", + { setupSessionId: cancelled.id }, + ); + } + } + } + } + function rejectEnvironmentDelete(input: { actor: ReturnType; environment: { id: string; driver: string }; @@ -824,38 +878,22 @@ export function environmentRoutes( res.status(404).json({ error: "Environment not found" }); return; } - - const hasStaticBlocker = impact.deleteBlockedReasons.some( - (r) => r === "managed_local" || r === "instance_default", - ); - if (hasStaticBlocker) { + const forceDelete = parseEnvironmentDeleteForce(req); + const staticDeleteBlockedReasons = impact.deleteBlockedReasons.filter((reason) => reason !== "active_runtime_use"); + if (staticDeleteBlockedReasons.length > 0) { rejectEnvironmentDelete({ actor, environment: existing, impact }); } - - const hasRuntimeBlocker = impact.deleteBlockedReasons.includes("active_runtime_use"); - const forceDelete = req.query.force === "true"; - if (hasRuntimeBlocker && !forceDelete) { - rejectEnvironmentDelete({ actor, environment: existing, impact }); - } - - if (hasRuntimeBlocker && forceDelete) { - const overview = await customImages.getOverview({ environmentId: existing.id }); - if (overview.activeSession) { - const cancelled = await customImages.cancelSetupSession({ - sessionId: overview.activeSession.id, - reason: "environment_force_delete", - }); - if (isActiveCustomImageSetupStatus(cancelled.status)) { - throw conflict( - "Active custom image setup session could not be cancelled. Retry or wait for setup to complete before deleting.", - { deleteBlockedReasons: ["active_runtime_use"] }, - ); - } + if (impact.activeRuntimeUse.hasActiveRuntimeUse) { + if (!forceDelete) { + rejectEnvironmentDeleteActiveRuntimeUse(impact); } - await svc.releaseActiveLeasesForEnvironment(existing.id); + await cleanupActiveRuntimeUseBeforeEnvironmentDelete({ + environmentId: existing.id, + impact, + }); } - const removed = await svc.removeIfDeletable(existing.id); + const removed = await svc.removeIfDeletableWithCleanup(existing.id); if (!removed) { const latestImpact = await svc.getDeleteBlastRadius(existing.id); if (!latestImpact) { @@ -864,29 +902,6 @@ export function environmentRoutes( } rejectEnvironmentDelete({ actor, environment: existing, impact: latestImpact }); } - const companyIds = await instanceSettings.listCompanyIds(); - await Promise.all( - companyIds.flatMap((companyId) => [ - executionWorkspaces.clearEnvironmentSelection(companyId, existing.id), - issues.clearExecutionWorkspaceEnvironmentSelection(companyId, existing.id), - projects.clearExecutionWorkspaceEnvironmentSelection(companyId, existing.id), - secrets.syncEnvBindingsForTarget( - companyId, - { targetType: "environment", targetId: existing.id }, - {}, - ), - secrets.syncSecretRefsForTarget( - companyId, - { targetType: "environment", targetId: existing.id }, - [], - { replaceAll: true }, - ), - ]), - ); - const secretId = readSshEnvironmentPrivateKeySecretId(existing); - if (secretId) { - await secrets.remove(secretId); - } await logInstanceEnvironmentActivity({ actor, action: "environment.deleted", diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index b37ee0508fb..61ce005f6dd 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -3747,7 +3747,10 @@ registry.registerPath({ path: "/api/environments/{id}", tags: ["environments"], summary: "Delete an environment", - request: { params: z.object({ id: z.string() }) }, + request: { + params: z.object({ id: z.string() }), + body: jsonBody(z.object({ force: z.boolean().optional() }).strict()), + }, responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden, 404: r.notFound, 409: r.conflict }, }); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 151b181a7c6..fccb162a0b0 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -1803,6 +1803,69 @@ export function environmentRuntimeService( return released; }, + async releaseActiveLeasesForEnvironment( + environmentId: string, + status: Extract = "released", + ): Promise { + const environment = await environmentsSvc.getById(environmentId); + if (!environment) return []; + const leaseRows = await db + .select() + .from(environmentLeases) + .where( + and( + eq(environmentLeases.environmentId, environmentId), + inArray(environmentLeases.status, ["active"]), + ), + ); + if (leaseRows.length === 0) return []; + + const released: EnvironmentRuntimeLeaseRecord[] = []; + for (const leaseRow of leaseRows) { + const leaseSnapshot = toEnvironmentLeaseSnapshot(leaseRow); + const driver = getDriver(getLeaseDriverKey(leaseSnapshot, environment)); + const lease = await (async () => { + if (!driver) { + return environmentsSvc.releaseLease(leaseSnapshot.id, "pending_cleanup", { + failureReason: "environment_delete_missing_runtime_driver", + cleanupStatus: "failed", + }); + } + if (leaseSnapshot.leasePolicy === "reuse_by_environment") { + if (!driver.destroyRunLease) { + return environmentsSvc.releaseLease(leaseSnapshot.id, "pending_cleanup", { + failureReason: "environment_delete_missing_reusable_lease_destroyer", + cleanupStatus: "failed", + }); + } + return driver.destroyRunLease({ + environment, + lease: leaseSnapshot, + failureReason: "environment_deleted", + }); + } + return driver.releaseRunLease({ + environment, + lease: leaseSnapshot, + status, + }); + })(); + if (!lease) continue; + + released.push({ + environment, + lease, + leaseContext: { + executionWorkspaceId: lease.executionWorkspaceId, + executionWorkspaceMode: + (lease.metadata?.executionWorkspaceMode as ExecutionWorkspace["mode"] | null | undefined) ?? null, + }, + }); + } + + return released; + }, + async destroyReusableSandboxLeases(input: { companyId: string; issueId?: string | null; diff --git a/server/src/services/environments.ts b/server/src/services/environments.ts index 4342a12a82d..b696544b4ed 100644 --- a/server/src/services/environments.ts +++ b/server/src/services/environments.ts @@ -1,7 +1,8 @@ -import { and, desc, eq, inArray, ne, sql } from "drizzle-orm"; +import { and, desc, eq, inArray, like, ne, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { agents, + companySecrets, companySecretBindings, environmentCustomImageSetupSessions, environmentLeases, @@ -10,6 +11,7 @@ import { instanceSettings, issues, projects, + userSecretDeclarations, } from "@paperclipai/db"; import { ENVIRONMENT_DRIVERS, @@ -180,6 +182,15 @@ function countFromRows(rows: Array<{ count: number | string | null | undefined } return Number(rows[0]?.count ?? 0); } +function readSshPrivateKeySecretIdFromEnvironment(row: EnvironmentRow): string | null { + if (row.driver !== "ssh") return null; + const config = cloneRecord(row.config, {}) ?? {}; + const ref = config.privateKeySecretRef; + if (!ref || typeof ref !== "object" || Array.isArray(ref)) return null; + const secretId = (ref as Record).secretId; + return typeof secretId === "string" && secretId.length > 0 ? secretId : null; +} + export function environmentService(db: Db) { return { list: async ( @@ -480,6 +491,92 @@ export function environmentService(db: Db) { return row ? toEnvironment(row) : null; }, + removeIfDeletableWithCleanup: async (id: string): Promise => { + const now = new Date(); + return db.transaction(async (tx) => { + const row = await tx + .delete(environments) + .where( + and( + eq(environments.id, id), + ne(environments.driver, "local"), + sql`not exists ( + select 1 from ${instanceSettings} + where ${instanceSettings.defaultEnvironmentId} = ${environments.id} + )`, + sql`not exists ( + select 1 from ${environmentLeases} + where ${environmentLeases.environmentId} = ${environments.id} + and ${environmentLeases.status} = 'active' + )`, + sql`not exists ( + select 1 from ${environmentCustomImageSetupSessions} + where ${environmentCustomImageSetupSessions.environmentId} = ${environments.id} + and ${environmentCustomImageSetupSessions.status} in ('starting', 'waiting_for_user', 'capturing') + )`, + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + if (!row) return null; + + await tx + .update(executionWorkspaces) + .set({ + metadata: sql`jsonb_set(coalesce(${executionWorkspaces.metadata}, '{}'::jsonb), '{config,environmentId}', 'null'::jsonb, true)`, + updatedAt: now, + }) + .where(sql`${executionWorkspaces.metadata} -> 'config' ->> 'environmentId' = ${id}`); + + await tx + .update(issues) + .set({ + executionWorkspaceSettings: + sql`jsonb_set(coalesce(${issues.executionWorkspaceSettings}, '{}'::jsonb), '{environmentId}', 'null'::jsonb, true)`, + updatedAt: now, + }) + .where(sql`${issues.executionWorkspaceSettings} ->> 'environmentId' = ${id}`); + + await tx + .update(projects) + .set({ + executionWorkspacePolicy: + sql`jsonb_set(coalesce(${projects.executionWorkspacePolicy}, '{}'::jsonb), '{environmentId}', 'null'::jsonb, true)`, + updatedAt: now, + }) + .where(sql`${projects.executionWorkspacePolicy} ->> 'environmentId' = ${id}`); + + await tx + .delete(userSecretDeclarations) + .where( + and( + eq(userSecretDeclarations.targetType, "environment"), + eq(userSecretDeclarations.targetId, id), + or( + eq(userSecretDeclarations.configPath, "env"), + like(userSecretDeclarations.configPath, "env.%"), + ), + ), + ); + + await tx + .delete(companySecretBindings) + .where( + and( + eq(companySecretBindings.targetType, "environment"), + eq(companySecretBindings.targetId, id), + ), + ); + + const privateKeySecretId = readSshPrivateKeySecretIdFromEnvironment(row); + if (privateKeySecretId) { + await tx.delete(companySecrets).where(eq(companySecrets.id, privateKeySecretId)); + } + + return toEnvironment(row); + }); + }, + getDeleteBlastRadius: async (id: string): Promise => { const environment = await db .select({ @@ -552,13 +649,16 @@ export function environmentService(db: Db) { const isManagedLocal = environment.driver === "local"; const isInstanceDefault = countFromRows(instanceDefaultRows) > 0; - const activeLeaseCount = countFromRows(activeLeaseRows); - const activeCustomImageSetupSessionCount = countFromRows(activeSetupRows); - const hasActiveRuntimeUse = activeLeaseCount > 0 || activeCustomImageSetupSessionCount > 0; const deleteBlockedReasons: EnvironmentDeleteBlockedReason[] = []; if (isManagedLocal) deleteBlockedReasons.push("managed_local"); if (isInstanceDefault) deleteBlockedReasons.push("instance_default"); - if (hasActiveRuntimeUse) deleteBlockedReasons.push("active_runtime_use"); + const activeLeaseCount = countFromRows(activeLeaseRows); + const activeCustomImageSetupSessionCount = countFromRows(activeSetupRows); + const hasActiveRuntimeUse = + activeLeaseCount > 0 || activeCustomImageSetupSessionCount > 0; + if (hasActiveRuntimeUse) { + deleteBlockedReasons.push("active_runtime_use"); + } return { environmentId: id, @@ -581,16 +681,6 @@ export function environmentService(db: Db) { }; }, - releaseActiveLeasesForEnvironment: async (id: string): Promise => { - const now = new Date(); - const rows = await db - .update(environmentLeases) - .set({ status: "released", releasedAt: now, lastUsedAt: now, updatedAt: now }) - .where(and(eq(environmentLeases.environmentId, id), eq(environmentLeases.status, "active"))) - .returning(); - return rows.length; - }, - listLeases: async ( environmentId: string, filters: {