From fcdceaf5fbe16b5fa1b98f36b0a0c8ec06f6c7aa Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:32:45 -0600 Subject: [PATCH] test(restart-chaos): grade a stopped retain-on-failure restore source as retained (#20086) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path - The restart-chaos harness's Daytona sweep grades R6, sandbox leaks, after every wave. It counts a sandbox as kept on purpose when its lease says so: a retained lease, or a released reusable lease whose sandbox is stopped for reuse. - Paperclip also keeps a failed run's sandbox stopped as a workspace-restore source (`retain_on_failure`, owned by restore recovery). The sweep graded those as orphans, so the warm-reuse chaos gate for #20048 counted intentional, recoverable retention as leaks. - The coordinator's decision: grade it honestly. A stopped, owned, recoverable restore source is retained; a running one, or one that survives its environment's deletion (no lease left), is still a leak. ## Linked Issues or Issue Description Refs #20048 **What happened?** In Daytona warm wave w-20261108T170423-s9501, four "orphans" were `retain_on_failure` leases: released, sandbox stopped, kept as failed runs' workspace copies. The sweep counted them as R6 leaks, failing the gate for intended retention. **Steps to reproduce** 1. Have a released `retain_on_failure` lease whose Daytona sandbox is stopped. 2. Run `reconcileSandboxes`: the sandbox is listed in `orphanSandboxes`. **Expected behavior** It is listed in `retainedLeases`. A running restore source, or a sandbox with no lease, stays in `orphanSandboxes`. ## What Changed - `tests/restart-chaos/daytona-sweep.ts`: `keptForReuse` also accepts a released `retain_on_failure` lease whose sandbox is `stopped`; the doc comment explains both kinds of kept sandbox. - `tests/restart-chaos/daytona-sweep.test.ts`: a stopped restore source is retained; a running one, or a sandbox with no lease, is still an orphan. ## Verification - `npx vitest run --root tests/restart-chaos daytona-sweep`: 14/14 pass. ## Risks - **Hiding a leak.** Only stopped sandboxes behind a released restore lease qualify; they are owned by restore recovery and, with #20085, destroyed with their environment. Running sandboxes and lease-less sandboxes still fail R6. This is harness-only, with no product change. ## Model Used Claude Opus 5.5 (`claude-opus-5-5`, Anthropic), 1M context window, tool use and code execution, via Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- tests/restart-chaos/daytona-sweep.test.ts | 11 +++++++++++ tests/restart-chaos/daytona-sweep.ts | 14 +++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/tests/restart-chaos/daytona-sweep.test.ts b/tests/restart-chaos/daytona-sweep.test.ts index 3569dd0ac6b..b2e6f91a9cc 100644 --- a/tests/restart-chaos/daytona-sweep.test.ts +++ b/tests/restart-chaos/daytona-sweep.test.ts @@ -126,6 +126,17 @@ describe("Daytona R6 reconciliation", () => { expect(result.orphanSandboxes.map(leak => leak.sandboxId)).toEqual(["r2", "e1"]); }); + it("keeps a stopped restore source (retain_on_failure) as retained, but not a running one or one with no lease", () => { + const result = reconcileSandboxes( + [{ id: "s1", state: "stopped" }, { id: "s2", state: "started" }, { id: "s3", state: "stopped" }], + [{ id: "f1", status: "released", leasePolicy: "retain_on_failure", providerLeaseId: "s1" }, + { id: "f2", status: "released", leasePolicy: "retain_on_failure", providerLeaseId: "s2" }], + ); + expect(result.retainedLeases.map(lease => lease.leaseId)).toEqual(["f1"]); + // Running, or left behind by a deleted environment (no lease): still leaks. + expect(result.orphanSandboxes.map(leak => leak.sandboxId)).toEqual(["s2", "s3"]); + }); + it("judges a reused sandbox by its current lease row, whatever the listing order", () => { const rows = [{ id: "old", status: "expired", leasePolicy: "reuse_by_environment", providerLeaseId: "r1" }, { id: "new", status: "released", leasePolicy: "reuse_by_environment", providerLeaseId: "r1" }]; diff --git a/tests/restart-chaos/daytona-sweep.ts b/tests/restart-chaos/daytona-sweep.ts index 9303335d8cb..d1d434ff51e 100644 --- a/tests/restart-chaos/daytona-sweep.ts +++ b/tests/restart-chaos/daytona-sweep.ts @@ -43,16 +43,24 @@ export interface DaytonaSweepResult { orphanSandboxes: SandboxLeak[]; /** Leases still `active` or `pending_cleanup` with every task at rest. */ heldLeases: LeaseLeak[]; - /** Leases kept on purpose (`retained`, or a released reusable lease whose sandbox is stopped for reuse), reported but not a leak. */ + /** Leases kept on purpose (`retained`, a released reusable lease whose sandbox is stopped for reuse, or a released retain-on-failure restore source whose sandbox is stopped), reported but not a leak. */ retainedLeases: LeaseLeak[]; /** Archived sandboxes behind a released lease: expected and reported; Daytona deletes them. */ archivedSandboxes: string[]; error: string | null; } -/** A stopped sandbox behind a released reuse_by_environment lease: a later run on the same workspace resumes it. */ +/** + * A stopped sandbox Paperclip keeps on purpose behind a released lease: + * - `reuse_by_environment`: a later run on the same workspace resumes it; + * - `retain_on_failure`: a failed run's workspace copy kept for file recovery + * (restore recovery owns it; environment deletion destroys it). + * Only a stopped one counts: a running sandbox, or one with no lease (for + * example after its environment was deleted), is still a leak. + */ function keptForReuse(lease: LeaseRow, sandbox: DaytonaSandboxRow): boolean { - return lease.status === "released" && lease.leasePolicy === "reuse_by_environment" + return lease.status === "released" + && (lease.leasePolicy === "reuse_by_environment" || lease.leasePolicy === "retain_on_failure") && String(sandbox.state ?? "").toLowerCase() === "stopped"; }