From bfa9002a3c6001260d68bc063d8b1ff0bf21e87f Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 8 Nov 2026 08:07:19 -0600 Subject: [PATCH] test(restart-chaos): reach the stop-and-retain crash point with --daytona-reuse-lease (#20049) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The restart-chaos crash-point suite kills the server at named points, then checks that recovery restores every invariant > - On Daytona, `environment.stop_retain.after_claim_before_stop` was never reached, because no workload produced a stop-and-retain release > - A per-turn run on a reusable sandbox lease ends every turn with `stop_and_reuse`, which releases through exactly that stop-only claim > - This pull request adds `--daytona-reuse-lease`, so the chaos Daytona environment keeps reusable sandboxes and the point is reachable > - The benefit is live coverage of the stop-and-retain crash window ## Linked Issues or Issue Description This is a follow-up from the Phase 2 durable-ownership data (`doc/plans/2026-11-07-phase2-reconciler-data.md`, "Not covered"). Related: #19937 (crash-point suite) and #20042 (stop-only claim fence renewal). No duplicate PR was found. **What happened?** The Daytona crash-point run `cp-20261108T104102` reported `environment.stop_retain.after_claim_before_stop` as `not_reached`. The point needs a stop-and-retain intent, and the chaos Daytona environment is created with ephemeral leases. **Expected behavior** A flag lets a Daytona wave or crash-point run use reusable leases. A per-turn run then stops and keeps its sandbox, and the point is reached and graded. **Steps to reproduce** 1. Run `node scripts/crash-points.mjs --env daytona --daytona-image --points environment.stop_retain.after_claim_before_stop --adapters codex --workloads A1`. 2. Before this PR the point is `not_reached`. 3. With `--daytona-reuse-lease --lifecycle per_turn` it is reached. **Paperclip version or commit** `master` at `8475826d2d` (the signed image `c24059f1…` revision). ## What Changed - `--daytona-reuse-lease` (wave and crash-point suite, Daytona only) creates the chaos environment with `reuseLease: true`. The crash-point suite passes the flag through. - The R6 sandbox sweep counts a **stopped** sandbox behind a **released `reuse_by_environment`** lease as kept for reuse, not a leak. A later run on the same workspace resumes it, because reuse accepts released and retained leases. A running sandbox, or a stopped one behind an ephemeral lease, is still a leak. - The crash-point registry prerequisite and the README name the flag. ## Verification - Live, Daytona, codex A1, from the image revision `8475826d2d`: - `cp-20261108T132149`: the point was **reached** and settled with no invariant violations. R6 flagged the stopped reusable sandbox as a leak, which led to the sweep fix. - `cp-20261108T133351`, with the sweep fix: **pass**. 0 orphaned sandboxes, 0 held leases, the environment deleted, and nothing left running. - `pnpm exec vitest run --config tests/restart-chaos/vitest.config.ts` passes `daytona-sweep`, `scheduler` and `crash-points` (52/52). `tsc -p tests/restart-chaos/tsconfig.json` is clean. ## Risks - Test harness only; the registry change is prerequisite text. Without the flag, waves behave as before. ## Model Used - Anthropic Claude Opus 5.5 (`claude-opus-5-5`), via Claude Code with tool use. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- packages/shared/src/crash-point-registry.ts | 2 +- tests/restart-chaos/README.md | 4 ++++ tests/restart-chaos/config.ts | 7 +++++-- tests/restart-chaos/crash-points.test.ts | 1 + tests/restart-chaos/crash-points.ts | 2 ++ tests/restart-chaos/daytona-sweep.test.ts | 21 ++++++++++++++++++++ tests/restart-chaos/daytona-sweep.ts | 22 +++++++++++++++++---- tests/restart-chaos/main.ts | 1 + tests/restart-chaos/scheduler.test.ts | 7 +++++++ tests/restart-chaos/seed.ts | 10 ++++++---- tests/restart-chaos/types.ts | 2 ++ 11 files changed, 68 insertions(+), 11 deletions(-) diff --git a/packages/shared/src/crash-point-registry.ts b/packages/shared/src/crash-point-registry.ts index 76313341d8e..65d2cacd0ec 100644 --- a/packages/shared/src/crash-point-registry.ts +++ b/packages/shared/src/crash-point-registry.ts @@ -113,7 +113,7 @@ export const CRASH_POINTS = { environments: ["daytona"], kinds: ["environment_stop_and_retain_intent"], invariants: ["I2", "I7"], - prerequisite: "A stop-and-retain intent: a run whose sandbox must be stopped but kept, for a native workspace export resume.", + prerequisite: "A stop-and-retain intent: run with --daytona-reuse-lease, so each per-turn run stops and keeps its reusable sandbox.", }, "environment.lease.after_release_before_runtime_services": { description: "The run's environment leases are released; its workspace runtime services are not.", diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index d2a36b3962f..87069b8bd18 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -287,6 +287,10 @@ release path) and `environment.stop_retain.after_claim_before_stop` (a stop-and-retain export) were not reached by any workload on 2026-11-08; a `not_reached` result prints what each needs. +`environment.stop_retain.after_claim_before_stop` needs `--daytona-reuse-lease`: +the chaos environment then keeps reusable sandboxes, so each per-turn run stops +and keeps its sandbox. + **CI.** `.github/workflows/crash-points.yml` runs the same command on demand (`workflow_dispatch`, inputs `points`, `kinds`, `adapters`, `workloads`, `nth`). It is not a required PR check. It runs only from the default branch, for diff --git a/tests/restart-chaos/config.ts b/tests/restart-chaos/config.ts index 0cdce2b67de..ff4196ba3c8 100644 --- a/tests/restart-chaos/config.ts +++ b/tests/restart-chaos/config.ts @@ -83,6 +83,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC replay: { type: "string" }, "daytona-image": { type: "string" }, "daytona-cpu": { type: "string" }, + "daytona-reuse-lease": { type: "boolean" }, keep: { type: "boolean" }, "rc06-masked": { type: "boolean" }, "storm-minutes": { type: "string" }, @@ -95,7 +96,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC const pollMs = integer(values["poll-ms"], "poll-ms", 3_000, 2_000, 5_000); const budgetCents = integer(values["budget-cents"], "budget-cents", 3_000, 1, 100_000); if (values.replay !== undefined) { - for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point"] as const) { + for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "daytona-reuse-lease", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point"] as const) { if (values[key] !== undefined) throw new Error(`--replay reproduces the saved plan; do not combine it with --${key}`); } return { mode: "replay", ref: parseReplayRef(values.replay), keep, @@ -140,7 +141,8 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC seed: integer(values.seed, "seed", now % 2_147_483_647, 0, 2_147_483_647), budgetCents, pollMs, - ...(env === "daytona" ? { daytonaImage, daytonaCpu: integer(values["daytona-cpu"], "daytona-cpu", DAYTONA_DEFAULT_CPU, 1, 4) } : {}), + ...(env === "daytona" ? { daytonaImage, daytonaCpu: integer(values["daytona-cpu"], "daytona-cpu", DAYTONA_DEFAULT_CPU, 1, 4), + ...(values["daytona-reuse-lease"] ? { daytonaReuseLease: true } : {}) } : {}), ...(values["rc06-masked"] ? { masks: ["rc06"] } : {}), ...(lifecycle ? { lifecycle } : {}), // Seeded K4 hold upper end; the controller-outage cap is 600 s (#19797). @@ -149,6 +151,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC ...(crashPoint ? { crashPoint } : {}), }; if (env !== "daytona" && values["daytona-cpu"] !== undefined) throw new Error("--daytona-cpu applies only to --env daytona"); + if (env !== "daytona" && values["daytona-reuse-lease"]) throw new Error("--daytona-reuse-lease applies only to --env daytona"); if (options.daytonaCpu !== undefined) { const budget = integer(process.env.RESTART_CHAOS_DAYTONA_VCPU_BUDGET, "RESTART_CHAOS_DAYTONA_VCPU_BUDGET", DAYTONA_VCPU_BUDGET, 1, 1_000); if (options.concurrency * options.daytonaCpu > budget) { diff --git a/tests/restart-chaos/crash-points.test.ts b/tests/restart-chaos/crash-points.test.ts index 575fa2bf0c8..daed8966922 100644 --- a/tests/restart-chaos/crash-points.test.ts +++ b/tests/restart-chaos/crash-points.test.ts @@ -41,6 +41,7 @@ describe("parseSuiteArgs", () => { it("passes harness options through and refuses unknown values", () => { expect(parseSuiteArgs(["--", "--keep", "--budget-cents", "500"]).passthrough).toEqual(["--budget-cents", "500", "--keep"]); + expect(parseSuiteArgs(["--env", "daytona", "--daytona-reuse-lease"]).passthrough).toEqual(["--daytona-reuse-lease"]); expect(() => parseSuiteArgs(["--adapters", "gpt"])).toThrow(/Unknown adapter/); expect(() => parseSuiteArgs(["--nth", "0"])).toThrow(/positive integer/); }); diff --git a/tests/restart-chaos/crash-points.ts b/tests/restart-chaos/crash-points.ts index 8255e258c8c..fdbb846105a 100644 --- a/tests/restart-chaos/crash-points.ts +++ b/tests/restart-chaos/crash-points.ts @@ -84,6 +84,7 @@ export function parseSuiteArgs(argv: readonly string[]): SuiteOptions { env: { type: "string" }, nth: { type: "string" }, seed: { type: "string" }, list: { type: "boolean" }, "daytona-image": { type: "string" }, "daytona-cpu": { type: "string" }, "budget-cents": { type: "string" }, "poll-ms": { type: "string" }, keep: { type: "boolean" }, lifecycle: { type: "string" }, + "daytona-reuse-lease": { type: "boolean" }, } }); const env = values.env ?? "local"; if (env !== "local" && env !== "daytona") throw new Error("--env must be local or daytona"); @@ -100,6 +101,7 @@ export function parseSuiteArgs(argv: readonly string[]): SuiteOptions { if (values[key] !== undefined) passthrough.push(`--${key}`, values[key]!); } if (values.keep) passthrough.push("--keep"); + if (values["daytona-reuse-lease"]) passthrough.push("--daytona-reuse-lease"); return { points: csv(values.points), kinds: csv(values.kinds), adapters, workloads, env, nth, seed, list: values.list === true, passthrough }; } diff --git a/tests/restart-chaos/daytona-sweep.test.ts b/tests/restart-chaos/daytona-sweep.test.ts index b75eea3865f..3569dd0ac6b 100644 --- a/tests/restart-chaos/daytona-sweep.test.ts +++ b/tests/restart-chaos/daytona-sweep.test.ts @@ -115,6 +115,27 @@ describe("Daytona R6 reconciliation", () => { }); + it("keeps a stopped sandbox behind a released reusable lease for reuse, but not a running one or an ephemeral one", () => { + const result = reconcileSandboxes( + [{ id: "r1", state: "stopped" }, { id: "r2", state: "started" }, { id: "e1", state: "stopped" }], + [{ id: "l1", status: "released", leasePolicy: "reuse_by_environment", providerLeaseId: "r1" }, + { id: "l2", status: "released", leasePolicy: "reuse_by_environment", providerLeaseId: "r2" }, + { id: "l3", status: "released", leasePolicy: "ephemeral", providerLeaseId: "e1" }], + ); + expect(result.retainedLeases.map(lease => lease.leaseId)).toEqual(["l1"]); + expect(result.orphanSandboxes.map(leak => leak.sandboxId)).toEqual(["r2", "e1"]); + }); + + it("judges a reused sandbox by its current lease row, whatever the listing order", () => { + const rows = [{ id: "old", status: "expired", leasePolicy: "reuse_by_environment", providerLeaseId: "r1" }, + { id: "new", status: "released", leasePolicy: "reuse_by_environment", providerLeaseId: "r1" }]; + for (const leases of [rows, [...rows].reverse()]) { + const result = reconcileSandboxes([{ id: "r1", state: "stopped" }], leases); + expect(result.orphanSandboxes).toEqual([]); + expect(result.retainedLeases.map(lease => lease.leaseId)).toEqual(["new"]); + } + }); + it("reports sandboxes with no lease and leases still active at rest", () => { const result = reconcileSandboxes( [{ id: "s9", state: "started", labels: { "paperclip-run-id": "r9" } }, { id: "s1", state: "started" }], diff --git a/tests/restart-chaos/daytona-sweep.ts b/tests/restart-chaos/daytona-sweep.ts index dc3b5eac958..9303335d8cb 100644 --- a/tests/restart-chaos/daytona-sweep.ts +++ b/tests/restart-chaos/daytona-sweep.ts @@ -18,7 +18,7 @@ import { deleteDaytonaEnvironment } from "../runner-e2e/live-fixtures.js"; import type { ChaosApi } from "./api.js"; export interface DaytonaSandboxRow { id: string; name?: string; state?: string; snapshot?: string; labels?: Record } -export interface LeaseRow { id: string; status: string; providerLeaseId?: string | null; heartbeatRunId?: string | null; issueId?: string | null } +export interface LeaseRow { id: string; status: string; leasePolicy?: string | null; providerLeaseId?: string | null; heartbeatRunId?: string | null; issueId?: string | null } /** Lease statuses where Paperclip still owns a provider resource. */ const HELD_LEASE_STATUSES = new Set(["active", "retained", "pending_cleanup"]); @@ -43,23 +43,37 @@ export interface DaytonaSweepResult { orphanSandboxes: SandboxLeak[]; /** Leases still `active` or `pending_cleanup` with every task at rest. */ heldLeases: LeaseLeak[]; - /** Leases kept on purpose (`retained`), reported but not a leak. */ + /** Leases kept on purpose (`retained`, or a released reusable lease whose sandbox is stopped for reuse), reported but not a leak. */ retainedLeases: LeaseLeak[]; /** Archived sandboxes behind a released lease: expected and reported; Daytona deletes them. */ archivedSandboxes: string[]; error: string | null; } +/** A stopped sandbox behind a released reuse_by_environment lease: a later run on the same workspace resumes it. */ +function keptForReuse(lease: LeaseRow, sandbox: DaytonaSandboxRow): boolean { + return lease.status === "released" && lease.leasePolicy === "reuse_by_environment" + && String(sandbox.state ?? "").toLowerCase() === "stopped"; +} + /** Pure reconciliation of one listing. */ export function reconcileSandboxes(sandboxes: readonly DaytonaSandboxRow[], leases: readonly LeaseRow[]): Pick { const live = sandboxes.filter(sandbox => !GONE_SANDBOX_STATES.has(String(sandbox.state ?? "").toLowerCase())); - const byProviderId = new Map(leases.filter(lease => lease.providerLeaseId).map(lease => [lease.providerLeaseId!, lease])); + const rank = (lease: LeaseRow) => HELD_LEASE_STATUSES.has(lease.status) ? 2 : lease.status === "released" ? 1 : 0; + const byProviderId = new Map(); + for (const lease of leases) { + if (!lease.providerLeaseId) continue; + const current = byProviderId.get(lease.providerLeaseId); + if (!current || rank(lease) > rank(current)) byProviderId.set(lease.providerLeaseId, lease); + } const orphanSandboxes: SandboxLeak[] = []; + const keptReusable: LeaseRow[] = []; const archivedSandboxes: string[] = []; for (const sandbox of live) { const lease = byProviderId.get(sandbox.id) ?? (sandbox.name ? byProviderId.get(sandbox.name) : undefined); if (lease && HELD_LEASE_STATUSES.has(lease.status)) continue; + if (lease && keptForReuse(lease, sandbox)) { keptReusable.push(lease); continue; } if (lease && ARCHIVED_SANDBOX_STATES.has(String(sandbox.state ?? "").toLowerCase())) { archivedSandboxes.push(sandbox.id); continue; } orphanSandboxes.push({ sandboxId: sandbox.id, state: sandbox.state ?? null, runId: sandbox.labels?.["paperclip-run-id"] ?? null, leaseId: lease?.id ?? null, leaseStatus: lease?.status ?? null, @@ -71,7 +85,7 @@ export function reconcileSandboxes(sandboxes: readonly DaytonaSandboxRow[], leas liveSandboxes: live.map(sandbox => sandbox.id), orphanSandboxes, heldLeases: leases.filter(lease => lease.status === "active" || lease.status === "pending_cleanup").map(toLeak), - retainedLeases: leases.filter(lease => lease.status === "retained").map(toLeak), + retainedLeases: [...leases.filter(lease => lease.status === "retained"), ...keptReusable].map(toLeak), archivedSandboxes, }; } diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index 2fdfec95e80..30ca3c0a045 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -206,6 +206,7 @@ async function main(): Promise { await supervisor.start(serverEnvironment); const api = new ChaosApi(supervisor.baseUrl); const lab = await seedChaosLab({ api, env: plan.options.env, daytonaImage: plan.options.daytonaImage, daytonaCpu: plan.options.daytonaCpu, + daytonaReuseLease: plan.options.daytonaReuseLease, adapters: [...new Set(plan.tasks.map(task => task.adapter))], concurrency: plan.options.concurrency, budgetCents: plan.options.budgetCents, masks: plan.options.masks, fixtureRoot: path.join(supervisor.paths.temporaryRoot, "fixtures"), waveId: plan.waveId, timeline, diff --git a/tests/restart-chaos/scheduler.test.ts b/tests/restart-chaos/scheduler.test.ts index 473825a52ef..b0b9bb0dcc1 100644 --- a/tests/restart-chaos/scheduler.test.ts +++ b/tests/restart-chaos/scheduler.test.ts @@ -168,6 +168,13 @@ describe("Daytona options", () => { expect(() => parseCommand(["--env", "daytona", "--daytona-image", image, "--concurrency", "11"])).toThrow(/exceeds the Daytona vCPU budget of 40/); expect(() => parseCommand(["--env", "daytona", "--daytona-image", "ghcr.io/x/y:latest"])).toThrow(/immutable runner image/); expect(() => parseCommand(["--env", "local", "--daytona-cpu", "2"])).toThrow(/only to --env daytona/); + expect(() => parseCommand(["--env", "local", "--daytona-reuse-lease"])).toThrow(/only to --env daytona/); + }); + it("keeps reusable Daytona leases only when asked", () => { + const reuse = parseCommand(["--env", "daytona", "--daytona-image", image, "--daytona-reuse-lease"]); + expect(reuse).toMatchObject({ mode: "wave", options: { daytonaReuseLease: true } }); + const plain = parseCommand(["--env", "daytona", "--daytona-image", image]); + if (plain.mode === "wave") expect(plain.options.daytonaReuseLease).toBeUndefined(); }); }); diff --git a/tests/restart-chaos/seed.ts b/tests/restart-chaos/seed.ts index 22cd68cd893..9c0b27044e7 100644 --- a/tests/restart-chaos/seed.ts +++ b/tests/restart-chaos/seed.ts @@ -58,7 +58,7 @@ export function chaosDaytonaShape(cpu: number) { return { cpu, memory: 4, disk: 10 }; } -async function createDaytonaEnvironment(input: { api: ChaosApi; companyId: string; image: string; cpu: number; waveId: string; +async function createDaytonaEnvironment(input: { api: ChaosApi; companyId: string; image: string; cpu: number; reuseLease: boolean; waveId: string; secretRefs: SecretReferenceMap; timeline: Timeline }): Promise { const fixture = runnerEnvironments.find(candidate => candidate.id === "daytona"); if (!fixture) throw new Error("runner-e2e catalog has no daytona environment fixture"); @@ -67,8 +67,9 @@ async function createDaytonaEnvironment(input: { api: ChaosApi; companyId: strin const payload = fixture.buildEnvironment({ secretRefs: input.secretRefs, daytonaImage: input.image, executionId: input.waveId }); const shape = chaosDaytonaShape(input.cpu); const environment = await input.api.post<{ id: string; driver: string }>(`/api/companies/${input.companyId}/environments`, - { ...payload, config: { ...(payload.config as Record), ...shape } }); - input.timeline.add("seed_environment", { environmentId: environment.id, driver: environment.driver, image: input.image, ...shape }); + { ...payload, config: { ...(payload.config as Record), ...shape, ...(input.reuseLease ? { reuseLease: true } : {}) } }); + input.timeline.add("seed_environment", { environmentId: environment.id, driver: environment.driver, image: input.image, ...shape, + reuseLease: input.reuseLease }); return environment.id; } @@ -108,6 +109,7 @@ export async function seedChaosLab(input: { env: EnvironmentKind; daytonaImage?: string; daytonaCpu?: number; + daytonaReuseLease?: boolean; adapters: readonly AdapterId[]; concurrency: number; budgetCents: number; @@ -158,7 +160,7 @@ export async function seedChaosLab(input: { let environmentId: string; if (input.env === "daytona") { environmentId = await createDaytonaEnvironment({ api, companyId: company.id, image: input.daytonaImage!, cpu: input.daytonaCpu ?? 4, - waveId: input.waveId, secretRefs, timeline }); + reuseLease: input.daytonaReuseLease === true, waveId: input.waveId, secretRefs, timeline }); input.onEnvironment?.({ companyId: company.id, environmentId, environmentKind: "daytona" }); if (input.adapters.includes("codex")) await qualifyNativeCodex({ api, companyId: company.id, environmentId, timeline }); } else { diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index da5d79b3f18..826882763a3 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -34,6 +34,8 @@ export interface WaveOptions { daytonaImage?: string; /** `--env daytona`: vCPU per sandbox (see seed.ts chaosDaytonaShape). */ daytonaCpu?: number; + /** `--daytona-reuse-lease`: the chaos environment keeps its sandboxes between turns (a per-turn run stops and keeps its sandbox). */ + daytonaReuseLease?: boolean; /** `--k4-hold-max-seconds`: upper end of the seeded K4 hold (default 300 s, cap 600 s). */ k4HoldMaxMs?: number; /**