diff --git a/server/src/__tests__/environment-routes.test.ts b/server/src/__tests__/environment-routes.test.ts index e7ed84da754..67db6a706d5 100644 --- a/server/src/__tests__/environment-routes.test.ts +++ b/server/src/__tests__/environment-routes.test.ts @@ -28,6 +28,7 @@ const mockProjectService = vi.hoisted(() => ({ const mockEnvironmentRuntimeService = vi.hoisted(() => ({ destroyReusableSandboxLeasesForEnvironment: vi.fn(async () => ({ destroyed: 0, failed: 0, skippedLiveRun: 0 })), + releaseRestoreSourcesForEnvironmentDelete: vi.fn(async () => 0), })); const mockCloseWarmNativeSessionsForEnvironment = vi.hoisted(() => vi.fn(async () => ({ closed: 0, busy: 0, failed: 0 })), @@ -1774,6 +1775,7 @@ describe("environment routes", () => { ).toHaveBeenCalledExactlyOnceWith({ environmentId: "env-1", failureReason: "environment_deleted", + consentedBy: "admin-1", }); expect( mockCloseWarmNativeSessionsForEnvironment, @@ -1845,6 +1847,12 @@ describe("environment routes", () => { expect(res.status).toBe(200); expect(retryPendingEnvironmentCleanup).toHaveBeenCalledExactlyOnceWith({ environmentId: "env-1", actorId: "admin-1" }); + // Consent releases restore-source protection before the cleanup retry, or a + // protected source would refuse every attempt and keep the delete blocked. + expect(mockEnvironmentRuntimeService.releaseRestoreSourcesForEnvironmentDelete) + .toHaveBeenCalledWith({ environmentId: "env-1", consentedBy: "admin-1" }); + expect(mockEnvironmentRuntimeService.releaseRestoreSourcesForEnvironmentDelete.mock.invocationCallOrder[0]) + .toBeLessThan(retryPendingEnvironmentCleanup.mock.invocationCallOrder[0]!); expect(mockEnvironmentService.removeIfDeletable).toHaveBeenCalledWith("env-1"); }); diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 5cb2daf5bdb..d2513eca283 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -5537,9 +5537,12 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { it("destroys a retained restore-source sandbox (retain_on_failure) when its environment is deleted", async () => { const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease(); await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, runId)); - // A failed run's sandbox kept, stopped, as a workspace-restore source. + // A failed run's sandbox kept, stopped, as a workspace-restore source. The + // restore marker protects it from every ordinary teardown. await db.update(environmentLeases).set({ leasePolicy: "retain_on_failure", status: "released", cleanupStatus: "success", - releasedAt: new Date() }).where(eq(environmentLeases.id, reusableLease.id)); + releasedAt: new Date(), metadata: { ...reusableLease.metadata, + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", runId } } }) + .where(eq(environmentLeases.id, reusableLease.id)); expect(await environmentService(db).getDeleteBlastRadius(reusableLease.environmentId!)) .toMatchObject({ reusableSandboxLeaseCount: 1, canDelete: false, reusableSandboxLeaseHolders: [expect.objectContaining({ leasePolicy: "retain_on_failure" })] }); @@ -5553,9 +5556,13 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; + // The operator consented to deleting the environment and the files only + // this sandbox holds: that consent releases the restore protection. const result = await environmentRuntimeService(db, { pluginWorkerManager: workerManager }) - .destroyReusableSandboxLeasesForEnvironment({ environmentId: reusableLease.environmentId!, failureReason: "environment_deleted" }); + .destroyReusableSandboxLeasesForEnvironment({ environmentId: reusableLease.environmentId!, failureReason: "environment_deleted", + consentedBy: "board-user" }); expect(result).toEqual({ destroyed: 1, failed: 0, skippedLiveRun: 0 }); + expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease", expect.anything(), expect.anything()); await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ status: "expired", failureReason: "environment_deleted", cleanupStatus: "success" }); }); diff --git a/server/src/routes/environments.ts b/server/src/routes/environments.ts index b73d3cb58e0..7acd2f1fddb 100644 --- a/server/src/routes/environments.ts +++ b/server/src/routes/environments.ts @@ -1343,6 +1343,18 @@ export function environmentRoutes( // own sandbox, already queued for teardown. Run one attempt now instead of // waiting out the retry cooldown. An attempt in flight elsewhere is not taken // over, and a failed attempt keeps the lease, so the re-check still rejects. + // Consent covers the environment's restore-source sandboxes too: release + // their restore protection first, so neither the pending-cleanup retry nor + // the teardown below refuses them (including ones an earlier attempt left + // claimed for cleanup). + if ( + req.query.destroyReusableSandboxLeases === "true" + && impact.deleteBlockedReasons.every((reason) => + reason === "pending_sandbox_cleanup" || reason === "reusable_sandbox_lease") + ) { + await environmentRuntime.releaseRestoreSourcesForEnvironmentDelete({ environmentId: existing.id, + consentedBy: actor.actorId }); + } if ( req.query.destroyReusableSandboxLeases === "true" && impact.pendingCleanupLeaseCount > 0 @@ -1378,6 +1390,7 @@ export function environmentRoutes( await environmentRuntime.destroyReusableSandboxLeasesForEnvironment({ environmentId: existing.id, failureReason: "environment_deleted", + consentedBy: actor.actorId, }); destroyedReusableSandboxLeaseCount = destroyResult.destroyed; impact = await svc.getDeleteBlastRadius(existing.id); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 96aba00bbb7..1e1a5d03de4 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -4309,12 +4309,30 @@ export function environmentRuntimeService( * throw leaves the lease in place — so the caller re-checks the blast * radius instead of trusting these counts for the delete decision. */ + /** + * The operator consented to deleting this environment and the files only + * its restore sources hold (the delete dialog says so). That consent is the + * decision that releases the restore protection: drop `workspaceRestoreRecovery` + * from every row on each of the environment's restore-source sandboxes, and + * record who released it, so the consented teardown can destroy them. + * Without this, every teardown refuses a protected restore source and the + * delete stays blocked with nothing the operator can do. + */ + async releaseRestoreSourcesForEnvironmentDelete(input: { environmentId: string; consentedBy: string }): Promise { + return await releaseRestoreSourcesForEnvironmentDelete(db, input); + }, + async destroyReusableSandboxLeasesForEnvironment(input: { environmentId: string; failureReason?: string; + /** The operator who consented to the delete; releases restore protection. */ + consentedBy?: string; }): Promise<{ destroyed: number; failed: number; skippedLiveRun: number }> { const environment = await environmentsSvc.getById(input.environmentId); if (!environment) return { destroyed: 0, failed: 0, skippedLiveRun: 0 }; + if (input.consentedBy) { + await releaseRestoreSourcesForEnvironmentDelete(db, { environmentId: input.environmentId, consentedBy: input.consentedBy }); + } const leaseRows = await db .select() .from(environmentLeases) @@ -4513,4 +4531,30 @@ export function environmentRuntimeService( }; } +async function releaseRestoreSourcesForEnvironmentDelete(db: Db, input: { environmentId: string; consentedBy: string }): Promise { + const sources = await db.select({ provider: environmentLeases.provider, providerLeaseId: environmentLeases.providerLeaseId }) + .from(environmentLeases).where(and( + eq(environmentLeases.environmentId, input.environmentId), + eq(environmentLeases.leasePolicy, "retain_on_failure"), + inArray(environmentLeases.status, ["released", "retained", "failed", "pending_cleanup"]), + sql`${environmentLeases.metadata} ? 'workspaceRestoreRecovery'`, + sql`${environmentLeases.providerLeaseId} is not null`, + )); + let released = 0; + for (const source of sources) { + const rows = await db.update(environmentLeases).set({ updatedAt: new Date(), + metadata: sql`(${environmentLeases.metadata} - 'workspaceRestoreRecovery') || ${JSON.stringify({ + restoreSourceReleasedByEnvironmentDelete: { consentedBy: input.consentedBy, releasedAt: new Date().toISOString() }, + })}::jsonb`, + }).where(and( + sql`${environmentLeases.provider} is not distinct from ${source.provider}`, + eq(environmentLeases.providerLeaseId, source.providerLeaseId!), + sql`${environmentLeases.metadata} ? 'workspaceRestoreRecovery'`, + )).returning({ id: environmentLeases.id }); + released += rows.length; + } + return released; +} + + export type EnvironmentRuntimeService = ReturnType;