From fc66ea0e0ae225a19bc5b6f4f85484d575805b09 Mon Sep 17 00:00:00 2001 From: Priya Raman Date: Thu, 9 Jul 2026 06:11:31 +0000 Subject: [PATCH] test: align delete blast radius Co-Authored-By: Paperclip --- .../src/__tests__/environment-routes.test.ts | 239 +++++++++++++-- .../src/__tests__/environment-service.test.ts | 278 +++++++++++++++++- 2 files changed, 488 insertions(+), 29 deletions(-) diff --git a/server/src/__tests__/environment-routes.test.ts b/server/src/__tests__/environment-routes.test.ts index c59ecc6f74e..a97bca1a3d9 100644 --- a/server/src/__tests__/environment-routes.test.ts +++ b/server/src/__tests__/environment-routes.test.ts @@ -35,6 +35,7 @@ const mockEnvironmentService = vi.hoisted(() => ({ create: vi.fn(), update: vi.fn(), removeIfDeletable: vi.fn(), + removeIfDeletableWithCleanup: vi.fn(), getDeleteBlastRadius: vi.fn(), listLeases: vi.fn(), getLeaseById: vi.fn(), @@ -77,6 +78,9 @@ const mockDeletePluginEnvironmentTemplate = vi.hoisted(() => vi.fn()); const mockExecutionWorkspaceService = vi.hoisted(() => ({ clearEnvironmentSelection: vi.fn(), })); +const mockEnvironmentRuntimeService = vi.hoisted(() => ({ + releaseActiveLeasesForEnvironment: vi.fn(), +})); vi.mock("../services/index.js", () => ({ issueService: () => mockIssueService, @@ -103,6 +107,10 @@ vi.mock("../services/execution-workspaces.js", () => ({ executionWorkspaceService: () => mockExecutionWorkspaceService, })); +vi.mock("../services/environment-runtime.js", () => ({ + environmentRuntimeService: () => mockEnvironmentRuntimeService, +})); + vi.mock("../services/plugin-environment-driver.js", () => ({ listReadyPluginEnvironmentDrivers: mockListReadyPluginEnvironmentDrivers, resolvePluginSandboxProviderDriverByKey: mockResolvePluginSandboxProviderDriverByKey, @@ -162,6 +170,7 @@ function createDeleteBlastRadius(overrides: Partial<{ const deleteBlockedReasons = [ ...(staticReferences.isManagedLocal ? ["managed_local" as const] : []), ...(staticReferences.isInstanceDefault ? ["instance_default" as const] : []), + ...(activeRuntimeUse.hasActiveRuntimeUse ? ["active_runtime_use" as const] : []), ]; return { environmentId: "env-1", @@ -234,10 +243,12 @@ describe("environment routes", () => { mockEnvironmentService.create.mockReset(); mockEnvironmentService.update.mockReset(); mockEnvironmentService.removeIfDeletable.mockReset(); + mockEnvironmentService.removeIfDeletableWithCleanup.mockReset(); mockEnvironmentService.getDeleteBlastRadius.mockReset(); mockEnvironmentService.listLeases.mockReset(); mockEnvironmentService.getLeaseById.mockReset(); mockExecutionWorkspaceService.clearEnvironmentSelection.mockReset(); + mockEnvironmentRuntimeService.releaseActiveLeasesForEnvironment.mockReset(); Object.values(mockEnvironmentCustomImageService).forEach((mock) => mock.mockReset()); mockEnvironmentCustomImageService.getOverview.mockResolvedValue({ activeTemplate: null, @@ -263,6 +274,7 @@ describe("environment routes", () => { mockIssueService.clearExecutionWorkspaceEnvironmentSelection.mockResolvedValue(0); mockProjectService.clearExecutionWorkspaceEnvironmentSelection.mockResolvedValue(0); mockExecutionWorkspaceService.clearEnvironmentSelection.mockResolvedValue(0); + mockEnvironmentRuntimeService.releaseActiveLeasesForEnvironment.mockResolvedValue([]); mockSecretService.normalizeEnvBindingsForPersistence.mockImplementation(async (_companyId, env) => env ?? {}); mockSecretService.listBindingCompanyIdsForTarget.mockResolvedValue([]); mockSecretService.syncEnvBindingsForTarget.mockResolvedValue([]); @@ -418,8 +430,8 @@ describe("environment routes", () => { expect(res.status).toBe(200); expect(res.body).toEqual({ environmentId: "env-1", - canDelete: true, - deleteBlockedReasons: [], + canDelete: false, + deleteBlockedReasons: ["active_runtime_use"], staticReferences: { isManagedLocal: false, isInstanceDefault: false, @@ -741,8 +753,8 @@ describe("environment routes", () => { expect(res.status).toBe(409); expect(res.body.error).toBe("Cannot delete the managed local environment."); expect(res.body.details).toEqual({ deleteBlockedReasons: ["managed_local"] }); - expect(mockEnvironmentService.removeIfDeletable).not.toHaveBeenCalled(); - expect(mockExecutionWorkspaceService.clearEnvironmentSelection).not.toHaveBeenCalled(); + expect(mockEnvironmentService.removeIfDeletableWithCleanup).not.toHaveBeenCalled(); + expect(mockEnvironmentRuntimeService.releaseActiveLeasesForEnvironment).not.toHaveBeenCalled(); }); it("rejects deleting the current instance default environment", async () => { @@ -778,10 +790,10 @@ describe("environment routes", () => { "Cannot delete the current instance default environment. Set a new default environment before deleting this one.", ); expect(res.body.details).toEqual({ deleteBlockedReasons: ["instance_default"] }); - expect(mockEnvironmentService.removeIfDeletable).not.toHaveBeenCalled(); + expect(mockEnvironmentService.removeIfDeletableWithCleanup).not.toHaveBeenCalled(); }); - it("clears environment selections and secret bindings across all companies when deleting an environment", async () => { + it("uses the atomic cleanup delete path when deleting an environment", async () => { const environment = { ...createEnvironment(), id: "env-ssh", @@ -804,7 +816,7 @@ describe("environment routes", () => { }; mockEnvironmentService.getById.mockResolvedValue(environment); mockEnvironmentService.getDeleteBlastRadius.mockResolvedValue(createDeleteBlastRadius()); - mockEnvironmentService.removeIfDeletable.mockResolvedValue(environment); + mockEnvironmentService.removeIfDeletableWithCleanup.mockResolvedValue(environment); mockInstanceSettingsService.listCompanyIds.mockResolvedValue(["company-1", "company-2"]); const app = createApp({ type: "board", @@ -815,27 +827,13 @@ describe("environment routes", () => { const res = await request(app).delete("/api/environments/env-ssh"); expect(res.status).toBe(200); - expect(mockEnvironmentService.removeIfDeletable).toHaveBeenCalledWith("env-ssh"); - for (const companyId of ["company-1", "company-2"]) { - expect(mockExecutionWorkspaceService.clearEnvironmentSelection) - .toHaveBeenCalledWith(companyId, "env-ssh"); - expect(mockIssueService.clearExecutionWorkspaceEnvironmentSelection) - .toHaveBeenCalledWith(companyId, "env-ssh"); - expect(mockProjectService.clearExecutionWorkspaceEnvironmentSelection) - .toHaveBeenCalledWith(companyId, "env-ssh"); - expect(mockSecretService.syncEnvBindingsForTarget).toHaveBeenCalledWith( - companyId, - { targetType: "environment", targetId: "env-ssh" }, - {}, - ); - expect(mockSecretService.syncSecretRefsForTarget).toHaveBeenCalledWith( - companyId, - { targetType: "environment", targetId: "env-ssh" }, - [], - { replaceAll: true }, - ); - } - expect(mockSecretService.remove).toHaveBeenCalledWith("11111111-1111-1111-1111-111111111111"); + expect(mockEnvironmentService.removeIfDeletableWithCleanup).toHaveBeenCalledWith("env-ssh"); + expect(mockExecutionWorkspaceService.clearEnvironmentSelection).not.toHaveBeenCalled(); + expect(mockIssueService.clearExecutionWorkspaceEnvironmentSelection).not.toHaveBeenCalled(); + expect(mockProjectService.clearExecutionWorkspaceEnvironmentSelection).not.toHaveBeenCalled(); + expect(mockSecretService.syncEnvBindingsForTarget).not.toHaveBeenCalled(); + expect(mockSecretService.syncSecretRefsForTarget).not.toHaveBeenCalled(); + expect(mockSecretService.remove).not.toHaveBeenCalled(); expect(mockLogActivity).toHaveBeenCalledWith( expect.anything(), expect.objectContaining({ @@ -847,6 +845,191 @@ describe("environment routes", () => { ); }); + it("requires force before deleting an environment with active runtime use", async () => { + const environment = { + ...createEnvironment(), + id: "env-sandbox", + name: "Sandbox Fixture", + driver: "sandbox" as const, + config: { provider: "fake", image: "paperclip-test", reuseLease: false }, + }; + mockEnvironmentService.getById.mockResolvedValue(environment); + mockEnvironmentService.getDeleteBlastRadius.mockResolvedValue(createDeleteBlastRadius({ + activeLeaseCount: 1, + activeCustomImageSetupSessionCount: 1, + })); + const app = createApp({ + type: "board", + userId: "admin-1", + source: "local_implicit", + }); + + const res = await request(app).delete("/api/environments/env-sandbox"); + + expect(res.status).toBe(409); + expect(res.body.error).toContain("Repeat the delete request with force: true"); + expect(res.body.details).toEqual({ + requiresForce: true, + activeRuntimeUse: { + activeLeaseCount: 1, + activeCustomImageSetupSessionCount: 1, + hasActiveRuntimeUse: true, + }, + }); + expect(mockEnvironmentRuntimeService.releaseActiveLeasesForEnvironment).not.toHaveBeenCalled(); + expect(mockEnvironmentService.removeIfDeletableWithCleanup).not.toHaveBeenCalled(); + }); + + it("cleans active runtime use before forced environment delete", async () => { + const environment = { + ...createEnvironment(), + id: "env-sandbox", + name: "Sandbox Fixture", + driver: "sandbox" as const, + config: { provider: "fake", image: "paperclip-test", reuseLease: false }, + }; + mockEnvironmentService.getById.mockResolvedValue(environment); + mockEnvironmentService.getDeleteBlastRadius.mockResolvedValue(createDeleteBlastRadius({ + activeLeaseCount: 1, + activeCustomImageSetupSessionCount: 1, + })); + mockEnvironmentRuntimeService.releaseActiveLeasesForEnvironment.mockResolvedValue([ + { + environment, + lease: { + id: "lease-1", + status: "released", + cleanupStatus: "success", + }, + leaseContext: { + executionWorkspaceId: null, + executionWorkspaceMode: null, + }, + }, + ]); + mockEnvironmentCustomImageService.getOverview.mockResolvedValue({ + activeTemplate: null, + activeSession: { + id: "setup-1", + environmentId: "env-sandbox", + provider: "secure-plugin", + status: "waiting_for_user", + }, + latestSession: null, + }); + mockEnvironmentCustomImageService.cancelSetupSession.mockResolvedValue({ + id: "setup-1", + environmentId: "env-sandbox", + provider: "secure-plugin", + status: "cancelled", + }); + mockEnvironmentService.removeIfDeletableWithCleanup.mockResolvedValue(environment); + const app = createApp({ + type: "board", + userId: "admin-1", + source: "local_implicit", + }); + + const res = await request(app) + .delete("/api/environments/env-sandbox") + .send({ force: true }); + + expect(res.status).toBe(200); + expect(mockEnvironmentRuntimeService.releaseActiveLeasesForEnvironment) + .toHaveBeenCalledWith("env-sandbox", "released"); + expect(mockEnvironmentCustomImageService.cancelSetupSession).toHaveBeenCalledWith({ + sessionId: "setup-1", + reason: "environment_deleted", + }); + expect(mockEnvironmentService.removeIfDeletableWithCleanup).toHaveBeenCalledWith("env-sandbox"); + }); + + it("rejects forced environment delete when active lease cleanup fails", async () => { + const environment = { + ...createEnvironment(), + id: "env-sandbox", + name: "Sandbox Fixture", + driver: "sandbox" as const, + config: { provider: "fake", image: "paperclip-test", reuseLease: false }, + }; + mockEnvironmentService.getById.mockResolvedValue(environment); + mockEnvironmentService.getDeleteBlastRadius.mockResolvedValue(createDeleteBlastRadius({ + activeLeaseCount: 1, + })); + mockEnvironmentRuntimeService.releaseActiveLeasesForEnvironment.mockResolvedValue([ + { + environment, + lease: { + id: "lease-1", + status: "released", + cleanupStatus: "failed", + }, + leaseContext: { + executionWorkspaceId: null, + executionWorkspaceMode: null, + }, + }, + ]); + const app = createApp({ + type: "board", + userId: "admin-1", + source: "local_implicit", + }); + + const res = await request(app) + .delete("/api/environments/env-sandbox") + .send({ force: true }); + + expect(res.status).toBe(409); + expect(res.body.error).toContain("could not clean up all active runtime leases"); + expect(res.body.details).toEqual({ failedLeaseIds: ["lease-1"] }); + expect(mockEnvironmentService.removeIfDeletableWithCleanup).not.toHaveBeenCalled(); + }); + + it("rejects forced environment delete when active custom image cancellation remains active", async () => { + const environment = { + ...createEnvironment(), + id: "env-sandbox", + name: "Sandbox Fixture", + driver: "sandbox" as const, + config: { provider: "fake", image: "paperclip-test", reuseLease: false }, + }; + mockEnvironmentService.getById.mockResolvedValue(environment); + mockEnvironmentService.getDeleteBlastRadius.mockResolvedValue(createDeleteBlastRadius({ + activeCustomImageSetupSessionCount: 1, + })); + mockEnvironmentRuntimeService.releaseActiveLeasesForEnvironment.mockResolvedValue([]); + mockEnvironmentCustomImageService.getOverview.mockResolvedValue({ + activeTemplate: null, + activeSession: { + id: "setup-1", + environmentId: "env-sandbox", + provider: "secure-plugin", + status: "waiting_for_user", + }, + latestSession: null, + }); + mockEnvironmentCustomImageService.cancelSetupSession.mockResolvedValue({ + id: "setup-1", + environmentId: "env-sandbox", + provider: "secure-plugin", + status: "waiting_for_user", + }); + const app = createApp({ + type: "board", + userId: "admin-1", + source: "local_implicit", + }); + + const res = await request(app) + .delete("/api/environments/env-sandbox") + .send({ force: true }); + + expect(res.status).toBe(409); + expect(res.body.error).toContain("could not cancel the active custom image setup session"); + expect(mockEnvironmentService.removeIfDeletableWithCleanup).not.toHaveBeenCalled(); + }); + it("rejects invalid SSH config on create", async () => { const app = createApp({ type: "board", diff --git a/server/src/__tests__/environment-service.test.ts b/server/src/__tests__/environment-service.test.ts index eff92293fd2..8a1f262efea 100644 --- a/server/src/__tests__/environment-service.test.ts +++ b/server/src/__tests__/environment-service.test.ts @@ -5,6 +5,7 @@ import { agents, companies, companySecretBindings, + companySecretVersions, companySecrets, createDb, environmentCustomImageSetupSessions, @@ -15,6 +16,8 @@ import { instanceSettings, issues, projects, + userSecretDeclarations, + userSecretDefinitions, } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, @@ -45,6 +48,7 @@ describeEmbeddedPostgres("environmentService leases", () => { afterEach(async () => { await db.delete(companySecretBindings); + await db.delete(userSecretDeclarations); await db.delete(environmentCustomImageSetupSessions); await db.delete(environmentLeases); await db.delete(heartbeatRuns); @@ -54,7 +58,9 @@ describeEmbeddedPostgres("environmentService leases", () => { await db.delete(agents); await db.delete(instanceSettings); await db.delete(environments); + await db.delete(companySecretVersions); await db.delete(companySecrets); + await db.delete(userSecretDefinitions); await db.delete(companies); }); @@ -381,7 +387,7 @@ describeEmbeddedPostgres("environmentService leases", () => { expect(impact).toEqual({ environmentId, canDelete: false, - deleteBlockedReasons: ["instance_default"], + deleteBlockedReasons: ["instance_default", "active_runtime_use"], staticReferences: { isManagedLocal: false, isInstanceDefault: true, @@ -473,6 +479,276 @@ describeEmbeddedPostgres("environmentService leases", () => { expect(deletedRows).toHaveLength(0); }); + it("removes an environment and cleanup references in one guarded transaction", async () => { + const companyId = randomUUID(); + const otherCompanyId = randomUUID(); + const environmentId = randomUUID(); + const projectId = randomUUID(); + const issueId = randomUUID(); + const workspaceId = randomUUID(); + const privateKeySecretId = randomUUID(); + const retainedSecretId = randomUUID(); + const otherTargetSecretId = randomUUID(); + const userSecretDefinitionId = randomUUID(); + const now = new Date(); + + await db.insert(companies).values([ + { + id: companyId, + name: "Acme", + status: "active", + issuePrefix: "ACM", + createdAt: now, + updatedAt: now, + }, + { + id: otherCompanyId, + name: "Other Co", + status: "active", + issuePrefix: "OTH", + createdAt: now, + updatedAt: now, + }, + ]); + await db.insert(environments).values({ + id: environmentId, + name: "Atomic SSH", + driver: "ssh", + status: "active", + config: { + host: "atomic.example.test", + port: 22, + username: "fixture", + remoteWorkspacePath: "/srv/paperclip", + privateKey: null, + privateKeySecretRef: { + type: "secret_ref", + secretId: privateKeySecretId, + version: "latest", + }, + knownHosts: null, + strictHostKeyChecking: true, + }, + createdAt: now, + updatedAt: now, + }); + await db.insert(projects).values({ + id: projectId, + companyId, + name: "Project", + status: "in_progress", + executionWorkspacePolicy: { + enabled: true, + defaultMode: "isolated_workspace", + environmentId, + }, + createdAt: now, + updatedAt: now, + }); + await db.insert(issues).values({ + id: issueId, + companyId, + projectId, + title: "Issue", + status: "todo", + priority: "medium", + executionWorkspaceSettings: { + mode: "isolated_workspace", + environmentId, + }, + createdAt: now, + updatedAt: now, + }); + await db.insert(executionWorkspaces).values({ + id: workspaceId, + companyId, + projectId, + sourceIssueId: issueId, + mode: "isolated_workspace", + strategyType: "git_worktree", + name: "Workspace", + status: "active", + providerType: "git_worktree", + metadata: { + config: { + environmentId, + }, + }, + createdAt: now, + updatedAt: now, + }); + await db.insert(companySecrets).values([ + { + id: privateKeySecretId, + companyId, + key: "atomic-private-key", + name: "Atomic Private Key", + provider: "local_encrypted", + createdAt: now, + updatedAt: now, + }, + { + id: retainedSecretId, + companyId, + key: "retained-env-secret", + name: "Retained Env Secret", + provider: "local_encrypted", + createdAt: now, + updatedAt: now, + }, + { + id: otherTargetSecretId, + companyId, + key: "other-target-secret", + name: "Other Target Secret", + provider: "local_encrypted", + createdAt: now, + updatedAt: now, + }, + ]); + await db.insert(companySecretVersions).values({ + secretId: privateKeySecretId, + version: 1, + material: { provider: "local_encrypted", ciphertext: "encrypted-private-key" }, + valueSha256: "private-key-value", + fingerprintSha256: "private-key-fingerprint", + createdAt: now, + }); + await db.insert(companySecretBindings).values([ + { + companyId, + secretId: privateKeySecretId, + targetType: "environment", + targetId: environmentId, + configPath: "config.privateKeySecretRef", + createdAt: now, + updatedAt: now, + }, + { + companyId, + secretId: retainedSecretId, + targetType: "environment", + targetId: environmentId, + configPath: "env.OPENAI_API_KEY", + createdAt: now, + updatedAt: now, + }, + { + companyId, + secretId: otherTargetSecretId, + targetType: "agent", + targetId: "agent-1", + configPath: "env.OPENAI_API_KEY", + createdAt: now, + updatedAt: now, + }, + ]); + await db.insert(userSecretDefinitions).values({ + id: userSecretDefinitionId, + companyId, + key: "deploy-key", + name: "Deploy Key", + provider: "local_encrypted", + createdAt: now, + updatedAt: now, + }); + await db.insert(userSecretDeclarations).values([ + { + companyId, + userSecretDefinitionId, + targetType: "environment", + targetId: environmentId, + configPath: "env.DEPLOY_KEY", + envKey: "DEPLOY_KEY", + createdAt: now, + updatedAt: now, + }, + { + companyId, + userSecretDefinitionId, + targetType: "agent", + targetId: "agent-1", + configPath: "env.DEPLOY_KEY", + envKey: "DEPLOY_KEY", + createdAt: now, + updatedAt: now, + }, + ]); + + const removed = await svc.removeIfDeletableWithCleanup(environmentId); + + expect(removed?.id).toBe(environmentId); + await expect(db.select().from(environments).where(eq(environments.id, environmentId))) + .resolves.toHaveLength(0); + const [workspace] = await db + .select({ metadata: executionWorkspaces.metadata }) + .from(executionWorkspaces) + .where(eq(executionWorkspaces.id, workspaceId)); + expect( + (workspace?.metadata as { config?: { environmentId?: unknown } } | null | undefined)?.config?.environmentId, + ).toBeNull(); + const [issue] = await db + .select({ executionWorkspaceSettings: issues.executionWorkspaceSettings }) + .from(issues) + .where(eq(issues.id, issueId)); + expect( + (issue?.executionWorkspaceSettings as { environmentId?: unknown } | null | undefined)?.environmentId, + ).toBeNull(); + const [project] = await db + .select({ executionWorkspacePolicy: projects.executionWorkspacePolicy }) + .from(projects) + .where(eq(projects.id, projectId)); + expect( + (project?.executionWorkspacePolicy as { environmentId?: unknown } | null | undefined)?.environmentId, + ).toBeNull(); + await expect( + db.select().from(companySecretBindings).where(and( + eq(companySecretBindings.targetType, "environment"), + eq(companySecretBindings.targetId, environmentId), + )), + ).resolves.toHaveLength(0); + await expect( + db.select().from(userSecretDeclarations).where(and( + eq(userSecretDeclarations.targetType, "environment"), + eq(userSecretDeclarations.targetId, environmentId), + )), + ).resolves.toHaveLength(0); + await expect(db.select().from(companySecrets).where(eq(companySecrets.id, privateKeySecretId))) + .resolves.toHaveLength(0); + await expect(db.select().from(companySecretVersions).where(eq(companySecretVersions.secretId, privateKeySecretId))) + .resolves.toHaveLength(0); + await expect(db.select().from(companySecrets).where(eq(companySecrets.id, retainedSecretId))) + .resolves.toHaveLength(1); + await expect(db.select().from(companySecretBindings).where(and( + eq(companySecretBindings.targetType, "agent"), + eq(companySecretBindings.targetId, "agent-1"), + ))).resolves.toHaveLength(1); + await expect(db.select().from(userSecretDeclarations).where(and( + eq(userSecretDeclarations.targetType, "agent"), + eq(userSecretDeclarations.targetId, "agent-1"), + ))).resolves.toHaveLength(1); + }); + + it("refuses cleanup delete when active runtime leases are present", async () => { + const { companyId, environmentId } = await seedEnvironment(); + await svc.acquireLease({ + companyId, + environmentId, + provider: "fake", + providerLeaseId: "provider-lease-1", + }); + + const removed = await svc.removeIfDeletableWithCleanup(environmentId); + + expect(removed).toBeNull(); + await expect(db.select().from(environments).where(eq(environments.id, environmentId))) + .resolves.toHaveLength(1); + await expect(db.select().from(environmentLeases).where(and( + eq(environmentLeases.environmentId, environmentId), + eq(environmentLeases.status, "active"), + ))).resolves.toHaveLength(1); + }); + it("creates and then reuses the default local environment for a company", async () => { const companyId = randomUUID(); await db.insert(companies).values({