From cefd17a82b63d2c4c428e165d2c19369efb57e41 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 8 Nov 2026 06:53:40 -0600 Subject: [PATCH] fix(environments): renew the stop-only cleanup claim while the provider stop runs (#20042) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - When a run ends with a "stop and retain" disposition, the control plane stops the agent's sandbox and keeps it for later recovery > - The release claims the lease with a 15-minute in-flight fence, so no cleanup sweep acts on it at the same time > - The stop-only path never renewed that fence; a provider stop that outlives 15 minutes lets a sweep take the lease over and stop it again mid-flight > - This pull request renews the stop-only claim every 30 s while the provider stop runs, like the destroy claim and the pending-cleanup sweep > - The benefit is one owner per cleanup attempt, however slow the provider is ## Linked Issues or Issue Description Follow-up gap from the Phase 2 ownership migration (durable ownership and reconciliation plan). Related: #19966 (teardown claim on the shared path), #19980 (pending-cleanup queue owner). No duplicate PR was found. **What happened?** `releaseRunLeases` with `providerResourceDisposition: "stop_and_retain"` claims the lease (`pendingCleanupInFlight: true`, `pendingCleanupLeaseExpiresAtMs: now + 15 min`) and calls the provider stop. Nothing renewed the fence while the stop ran. The destroy claim and the pending-cleanup sweep both renew theirs every 30 s. **Expected behavior** While this process is still running the provider stop, its claim stays fenced, so no other sweep starts a second attempt. **Steps to reproduce** 1. Release a run lease with `stop_and_retain` against a provider whose stop RPC blocks. 2. Let more than 15 minutes pass. 3. See that the fence expired while the first stop is still in flight. **Paperclip version or commit** `master` before this PR. ## What Changed - New `renewCleanupClaim(leaseId, attemptId, failure)` in `environment-runtime.ts`. It renews `pendingCleanupLeaseExpiresAtMs` every 30 s, only while the row still names this attempt and is in flight. - The destroy claim now uses the helper; its behaviour is unchanged. - The stop-only claim starts the renewal right after the claim and stops it before it settles the attempt, on success, on failure and in `finally`. ## Verification - Red, before the change: `environment-runtime.test.ts` "renews the stop-only cleanup claim while the provider stop is still running" failed. The fence stayed at its aged value. - Green: `pnpm exec vitest run src/__tests__/environment-runtime.test.ts src/__tests__/heartbeat-pending-cleanup-sweep.test.ts`, 247/247. ## Risks - Low. One extra metadata write every 30 s per in-flight stop, the same as for the destroy claim. The renewal is fenced on the attempt id, so it never extends another attempt's claim. ## Model Used - Anthropic Claude Opus 5.5 (`claude-opus-5-5`), via Claude Code with tool use. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- .../src/__tests__/environment-runtime.test.ts | 30 ++++++++++++- server/src/services/environment-runtime.ts | 42 ++++++++++++------- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 79cc2b987b1..34c9b18aa09 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -10,7 +10,7 @@ import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import { buildSshEnvLabFixtureConfig, getSshEnvLabSupport, @@ -1126,6 +1126,34 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { }, ); + it("renews the stop-only cleanup claim while the provider stop is still running", async () => { + const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); + const lease = seeded.reusableLease; + await db.update(environmentLeases).set({ leasePolicy: "ephemeral", metadata: { ...lease.metadata, reuseLease: false } }) + .where(eq(environmentLeases.id, lease.id)); + await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId)); + const started = Promise.withResolvers(), finish = Promise.withResolvers(); + const call = vi.fn(async (_id: string, method: string) => { + if (method === "environmentStopLease") { started.resolve(); await finish.promise; } + return { providerLeaseId: lease.providerLeaseId, state: "stopped" }; + }); + const runtime = environmentRuntimeService(db, { pluginWorkerManager: { + isRunning: () => true, call, + getWorker: () => ({ supportedMethods: ["environmentReleaseLease", "environmentDestroyLease", "environmentStopLease"] }), + } as unknown as PluginWorkerManager }); + vi.useFakeTimers({ toFake: ["setInterval", "clearInterval"] }); + const releasing = runtime.releaseRunLeases(seeded.runId, "failed", undefined, "stop_and_retain"); + try { + await started.promise; + const expiry = async () => Number((await environmentService(db).getLeaseById(lease.id))?.metadata?.pendingCleanupLeaseExpiresAtMs); + await db.update(environmentLeases).set({ metadata: sql`${environmentLeases.metadata} || '{"pendingCleanupLeaseExpiresAtMs":1}'::jsonb` }) + .where(eq(environmentLeases.id, lease.id)); + await vi.advanceTimersByTimeAsync(30_000); + await vi.waitFor(async () => expect(await expiry()).toBeGreaterThan(Date.now() + 14 * 60_000)); + } finally { finish.resolve(); await releasing; vi.useRealTimers(); } + expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "released", cleanupStatus: "success" }); + }); + it("reports an unwired manager separately from a stopped sandbox worker", async () => { const { companyId, environment, runId } = await seedReusablePluginSandboxLease(); const input = { diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index ba7c75b7391..2a9895ece08 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -3697,6 +3697,26 @@ export function environmentRuntimeService( return driver; } + function renewCleanupClaim(leaseId: string, attemptId: string, failure: string): ReturnType { + let renewalInFlight = false; + const renewal = setInterval(() => { + if (renewalInFlight) return; + renewalInFlight = true; + void db.update(environmentLeases).set({ + metadata: sql`${environmentLeases.metadata} || ${JSON.stringify({ + pendingCleanupLeaseExpiresAtMs: Date.now() + 15 * 60_000, + })}::jsonb`, + }).where(and(eq(environmentLeases.id, leaseId), eq(environmentLeases.status, "pending_cleanup"), + sql`${environmentLeases.metadata}->>'pendingCleanupAttemptId' = ${attemptId}`, + sql`${environmentLeases.metadata}->>'pendingCleanupInFlight' = 'true'`, + )).then(() => {}).catch(() => { + logger.warn({ leaseId }, failure); + }).finally(() => { renewalInFlight = false; }); + }, 30_000); + renewal.unref(); + return renewal; + } + async function destroyClaimedSandboxLease(input: { environment: Environment; leaseRow: typeof environmentLeases.$inferSelect; @@ -3747,22 +3767,7 @@ export function environmentRuntimeService( if (!claimed) return null; crashPoint("environment.teardown.after_claim_before_destroy"); const lease = toEnvironmentLeaseSnapshot(claimed); - let renewalInFlight = false; - const renewal = setInterval(() => { - if (renewalInFlight) return; - renewalInFlight = true; - void db.update(environmentLeases).set({ - metadata: sql`${environmentLeases.metadata} || ${JSON.stringify({ - pendingCleanupLeaseExpiresAtMs: Date.now() + 15 * 60_000, - })}::jsonb`, - }).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.status, "pending_cleanup"), - sql`${environmentLeases.metadata}->>'pendingCleanupAttemptId' = ${attemptId}`, - sql`${environmentLeases.metadata}->>'pendingCleanupInFlight' = 'true'`, - )).then(() => {}).catch(() => { - logger.warn({ leaseId: lease.id }, "scoped cleanup ownership renewal failed"); - }).finally(() => { renewalInFlight = false; }); - }, 30_000); - renewal.unref(); + const renewal = renewCleanupClaim(lease.id, attemptId, "scoped cleanup ownership renewal failed"); try { const driver = getDriver(getLeaseDriverKey(lease, input.environment)); if (!driver?.destroyRunLease) return lease; @@ -3984,16 +3989,21 @@ export function environmentRuntimeService( if (!claimed) continue; crashPoint("environment.stop_retain.after_claim_before_stop"); const snapshot = toEnvironmentLeaseSnapshot(claimed); + const renewal = renewCleanupClaim(snapshot.id, attemptId, "stop-only cleanup ownership renewal failed"); try { const driver = requireDriverKey(getLeaseDriverKey(snapshot, environment)); if (!driver.retryPendingSandboxTeardown) throw new Error("Workspace export recovery requires stop-only cleanup."); const receipt = await driver.retryPendingSandboxTeardown({ environment, lease: snapshot }); + clearInterval(renewal); const lease = await settleStopOnlyCleanup(db, snapshot, { attemptId, receipt }); if (lease && environment) released.push({ environment, lease: toEnvironmentLeaseSnapshot(lease), leaseContext: { executionWorkspaceId: lease.executionWorkspaceId, executionWorkspaceMode: null } }); } catch (error) { + clearInterval(renewal); await settleStopOnlyCleanup(db, snapshot, { attemptId }); throw error; + } finally { + clearInterval(renewal); } continue; }