mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-12 14:19:00 +01:00
fix(environments): renew the stop-only cleanup claim while the provider stop runs (#20042)
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - When a run ends with a "stop and retain" disposition, the control plane stops the agent's sandbox and keeps it for later recovery > - The release claims the lease with a 15-minute in-flight fence, so no cleanup sweep acts on it at the same time > - The stop-only path never renewed that fence; a provider stop that outlives 15 minutes lets a sweep take the lease over and stop it again mid-flight > - This pull request renews the stop-only claim every 30 s while the provider stop runs, like the destroy claim and the pending-cleanup sweep > - The benefit is one owner per cleanup attempt, however slow the provider is ## Linked Issues or Issue Description Follow-up gap from the Phase 2 ownership migration (durable ownership and reconciliation plan). Related: #19966 (teardown claim on the shared path), #19980 (pending-cleanup queue owner). No duplicate PR was found. **What happened?** `releaseRunLeases` with `providerResourceDisposition: "stop_and_retain"` claims the lease (`pendingCleanupInFlight: true`, `pendingCleanupLeaseExpiresAtMs: now + 15 min`) and calls the provider stop. Nothing renewed the fence while the stop ran. The destroy claim and the pending-cleanup sweep both renew theirs every 30 s. **Expected behavior** While this process is still running the provider stop, its claim stays fenced, so no other sweep starts a second attempt. **Steps to reproduce** 1. Release a run lease with `stop_and_retain` against a provider whose stop RPC blocks. 2. Let more than 15 minutes pass. 3. See that the fence expired while the first stop is still in flight. **Paperclip version or commit** `master` before this PR. ## What Changed - New `renewCleanupClaim(leaseId, attemptId, failure)` in `environment-runtime.ts`. It renews `pendingCleanupLeaseExpiresAtMs` every 30 s, only while the row still names this attempt and is in flight. - The destroy claim now uses the helper; its behaviour is unchanged. - The stop-only claim starts the renewal right after the claim and stops it before it settles the attempt, on success, on failure and in `finally`. ## Verification - Red, before the change: `environment-runtime.test.ts` "renews the stop-only cleanup claim while the provider stop is still running" failed. The fence stayed at its aged value. - Green: `pnpm exec vitest run src/__tests__/environment-runtime.test.ts src/__tests__/heartbeat-pending-cleanup-sweep.test.ts`, 247/247. ## Risks - Low. One extra metadata write every 30 s per in-flight stop, the same as for the destroy claim. The renewal is fenced on the attempt id, so it never extends another attempt's claim. ## Model Used - Anthropic Claude Opus 5.5 (`claude-opus-5-5`), via Claude Code with tool use. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c966334ec3
commit
cefd17a82b
2 files changed
+55
-17
No files matched your search
@@ -10,7 +10,7 @@ import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import {
|
||||
buildSshEnvLabFixtureConfig,
|
||||
getSshEnvLabSupport,
|
||||
@@ -1126,6 +1126,34 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("renews the stop-only cleanup claim while the provider stop is still running", async () => {
|
||||
const seeded = await seedReusablePluginSandboxLease("paperclip_runner");
|
||||
const lease = seeded.reusableLease;
|
||||
await db.update(environmentLeases).set({ leasePolicy: "ephemeral", metadata: { ...lease.metadata, reuseLease: false } })
|
||||
.where(eq(environmentLeases.id, lease.id));
|
||||
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId));
|
||||
const started = Promise.withResolvers<void>(), finish = Promise.withResolvers<void>();
|
||||
const call = vi.fn(async (_id: string, method: string) => {
|
||||
if (method === "environmentStopLease") { started.resolve(); await finish.promise; }
|
||||
return { providerLeaseId: lease.providerLeaseId, state: "stopped" };
|
||||
});
|
||||
const runtime = environmentRuntimeService(db, { pluginWorkerManager: {
|
||||
isRunning: () => true, call,
|
||||
getWorker: () => ({ supportedMethods: ["environmentReleaseLease", "environmentDestroyLease", "environmentStopLease"] }),
|
||||
} as unknown as PluginWorkerManager });
|
||||
vi.useFakeTimers({ toFake: ["setInterval", "clearInterval"] });
|
||||
const releasing = runtime.releaseRunLeases(seeded.runId, "failed", undefined, "stop_and_retain");
|
||||
try {
|
||||
await started.promise;
|
||||
const expiry = async () => Number((await environmentService(db).getLeaseById(lease.id))?.metadata?.pendingCleanupLeaseExpiresAtMs);
|
||||
await db.update(environmentLeases).set({ metadata: sql`${environmentLeases.metadata} || '{"pendingCleanupLeaseExpiresAtMs":1}'::jsonb` })
|
||||
.where(eq(environmentLeases.id, lease.id));
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
await vi.waitFor(async () => expect(await expiry()).toBeGreaterThan(Date.now() + 14 * 60_000));
|
||||
} finally { finish.resolve(); await releasing; vi.useRealTimers(); }
|
||||
expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "released", cleanupStatus: "success" });
|
||||
});
|
||||
|
||||
it("reports an unwired manager separately from a stopped sandbox worker", async () => {
|
||||
const { companyId, environment, runId } = await seedReusablePluginSandboxLease();
|
||||
const input = {
|
||||
|
||||
Reference in new issue
Block a user