From e67f3bd63a99e68f9fc56b427747b65b1d0dc865 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:42:08 -0600 Subject: [PATCH] test(environments): add an opt-in live Daytona proof for warm sandbox expiry (#21463) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Agents run in remote sandboxes, such as Daytona, through reusable environment leases > - #21410 made Paperclip stop a warm sandbox when its warm window ends > - Its tests use a real database but a mocked provider stop, and staging had no warm reusable sandboxes to observe > - This pull request adds an opt-in live test that proves the behavior against real Daytona > - The benefit is a repeatable live proof of warm expiry, resume, and recovery after a restart ## Linked Issues or Issue Description Refs #21410 **What happened?** #21410 has no live provider proof. Staging creates no warm reusable sandboxes today, so the deployed change cannot be observed there. **Expected behavior** A repeatable live check shows that a warm sandbox stays started inside its window and stops after it. It also shows that the next turn resumes the same sandbox, and that a restarted cleanup worker still stops the sandbox on time. **Steps to reproduce** 1. Set `PAPERCLIP_LIVE_WARM_EXPIRY=1` and `DAYTONA_API_KEY`. 2. Run `pnpm exec vitest run src/__tests__/warm-sandbox-expiry.live.test.ts` in `server`. **Additional context** The test makes no model turn. It drives the production environment runtime and the cleanup worker against real Daytona sandboxes. ## What Changed - New `server/src/__tests__/warm-sandbox-expiry.live.test.ts`. The test is skipped unless `PAPERCLIP_LIVE_WARM_EXPIRY=1` is set. It uses: - the real Daytona plugin and real sandboxes; - `environmentRuntime.acquireRunLease` with a 2 minute warm window; - `releaseRunLeases(..., "keep_running")`; - the production `registerEnvironmentCleanupWork` on a real `createDeliveryWorkCoordinator`, which must wake itself at the deadline. - The test checks the following: - The sandbox is started 20 s before the window ends. - After the window, the worker stops the sandbox. The sandbox is stopped, not deleted, and the lease is released as stopped and kept. - The second turn resumes the same sandbox (`acquisition: resumed`). - A new database client, a freshly loaded plugin module, and a new cleanup worker start in the middle of the window. That worker stops the sandbox after the window ends, not before. - At the end, the test deletes only the sandboxes whose labels match its own company and environment. ## Verification - Live run on 2026-11-13 against the Daytona org, window 120 s, sandbox `170bb215…`: - 18:39:32.624Z turn 1 lease retained; window ends 18:41:32.624Z - 18:41:13Z sandbox `started` (20 s before the window ends) - 18:41:36Z sandbox `stopped`, auto-delete -1: the worker stopped it 4 s after the window ended - 18:41:39Z turn 2 resumed the same sandbox (`resumed`, `started`); window ends 18:43:39.549Z - 18:42:10Z restart: a new worker booted; sandbox still `started` - 18:43:45Z sandbox `stopped`, 5 s after the window ended - 18:43:45Z fixture sandbox deleted; Daytona returns 404 - Provider calls: one acquire, one resume, two stops, no destroy before cleanup - Without the env flag the test is skipped (1 skipped). ## Risks - Low risk. The change adds a test file only, and the test is skipped by default. It makes real provider calls only when someone opts in. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, used through Claude Code with tool use (shell, file edits, Daytona API). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../warm-sandbox-expiry.live.test.ts | 229 ++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 server/src/__tests__/warm-sandbox-expiry.live.test.ts diff --git a/server/src/__tests__/warm-sandbox-expiry.live.test.ts b/server/src/__tests__/warm-sandbox-expiry.live.test.ts new file mode 100644 index 00000000000..d4a022959b1 --- /dev/null +++ b/server/src/__tests__/warm-sandbox-expiry.live.test.ts @@ -0,0 +1,229 @@ +// Opt-in live provider proof for warm sandbox expiry. Not a Product E2E +// workflow and no model turn: real Daytona sandboxes, the real Daytona plugin, +// the production environment runtime (acquire, warm retain, resume), and the +// production environment cleanup worker on a real delivery coordinator, which +// must wake itself at the warm-window deadline. +// +// Prerequisite: the Daytona provider is outside the root workspace, so install +// and build it on its own first (this test loads its dist/ and its own SDK): +// pnpm --dir packages/plugins/sandbox-providers/daytona install +// pnpm --dir packages/plugins/sandbox-providers/daytona build +// Then, from server/: +// PAPERCLIP_LIVE_WARM_EXPIRY=1 DAYTONA_API_KEY=... \ +// pnpm exec vitest run src/__tests__/warm-sandbox-expiry.live.test.ts +import { randomUUID } from "node:crypto"; +import { createRequire } from "node:module"; +import { access, writeFile } from "node:fs/promises"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, companies, createDb, environmentLeases, environments, heartbeatRuns, issues, plugins } from "@paperclipai/db"; +import type { Environment } from "@paperclipai/shared"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { nextEnvironmentCleanupAt } from "../services/environment-cleanup-state.js"; +import { registerEnvironmentCleanupWork } from "../services/environment-cleanup-work.js"; +import { environmentRuntimeService } from "../services/environment-runtime.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; + +const enabled = process.env.PAPERCLIP_LIVE_WARM_EXPIRY === "1"; +const describeLive = enabled ? describe : describe.skip; +const WINDOW_MS = Number(process.env.PAPERCLIP_LIVE_WARM_EXPIRY_WINDOW_MS ?? 120_000); +const pluginUrl = new URL("../../../packages/plugins/sandbox-providers/daytona/dist/plugin.js", import.meta.url).href; +const manifestUrl = new URL("../../../packages/plugins/sandbox-providers/daytona/dist/manifest.js", import.meta.url).href; +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, Math.max(0, ms))); +const iso = (ms: number) => new Date(ms).toISOString(); + +describeLive("live Daytona warm sandbox expiry", () => { + let temporary: Awaited>; + let db: ReturnType; + let plugin: any; + let manifest: any; + let daytona: any; + let environment: Environment; + const sandboxIds = new Set(); + const ids = { company: randomUUID(), agent: randomUUID(), environment: randomUUID(), issue: randomUUID(), plugin: randomUUID() }; + const config = { provider: "daytona", reuseLease: true, runnerLifecycleMode: "warm", runnerIdleTimeoutMs: WINDOW_MS, + timeoutMs: 180_000, autoStopInterval: 30, autoArchiveInterval: 60, autoDeleteInterval: 120 }; + const timeline: Array> = []; + const methods: string[] = []; + const supported = ["environmentValidateConfig", "environmentProbe", "environmentAcquireLease", "environmentResumeLease", + "environmentReleaseLease", "environmentStopLease", "environmentDestroyLease", "environmentRealizeWorkspace", + "environmentExecute", "environmentInspectLease"]; + const manager = () => ({ + isRunning: () => true, + getWorker: () => ({ supportedMethods: supported }), + call: async (_id: string, method: string, input: any) => { + methods.push(method); + const handler = `on${method[0]!.toUpperCase()}${method.slice(1)}`; + if (typeof plugin[handler] !== "function") throw new Error(`Unsupported provider operation ${method}`); + return await plugin[handler](input); + }, + }) as unknown as PluginWorkerManager; + + async function provider(id: string) { + const sandbox = await daytona.get(id); + await sandbox.refreshData(); + return { state: sandbox.state as string, autoDeleteInterval: sandbox.autoDeleteInterval as number, + labels: sandbox.labels as Record, updatedAt: sandbox.updatedAt as string }; + } + async function note(step: string, providerLeaseId: string, extra: Record = {}) { + const observed = await provider(providerLeaseId); + const entry = { at: new Date().toISOString(), step, sandboxState: observed.state, + sandboxAutoDelete: observed.autoDeleteInterval, ...extra }; + timeline.push(entry); + console.log(JSON.stringify(entry)); + return observed; + } + async function run(status: string) { + const id = randomUUID(); + await db.insert(heartbeatRuns).values({ id, companyId: ids.company, agentId: ids.agent, issueId: ids.issue, + invocationSource: "on_demand", status, nextEventSeq: 1 }); + return id; + } + async function leaseRows() { + return db.select().from(environmentLeases).where(eq(environmentLeases.companyId, ids.company)); + } + /** One warm turn: acquire the reusable lease through the production runtime, + * then end the run the way a succeeded warm run does (keep_running). */ + async function warmTurn(label: string) { + const runId = await run("running"); + const runtime = environmentRuntimeService(db, { pluginWorkerManager: manager() }); + const acquired = await runtime.acquireRunLease({ companyId: ids.company, environment, issueId: ids.issue, + agentId: ids.agent, heartbeatRunId: runId, persistedExecutionWorkspace: null, adapterType: "paperclip_runner", warmRunnerIdleTimeoutMs: WINDOW_MS }); + const providerLeaseId = acquired.lease.providerLeaseId!; + sandboxIds.add(providerLeaseId); + await note(`${label}: acquired`, providerLeaseId, { providerLeaseId, + acquisition: (acquired.lease.metadata?.sandboxLeaseAcquisition as { outcome?: string } | undefined)?.outcome, + recordedWindowMs: acquired.lease.metadata?.warmSandboxIdleTimeoutMs }); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + const released = await runtime.releaseRunLeases(runId, "released", undefined, "keep_running"); + expect(released).toHaveLength(1); + const [retained] = (await leaseRows()).filter(row => row.heartbeatRunId === runId); + expect(retained).toMatchObject({ status: "retained", leasePolicy: "reuse_by_environment", providerLeaseId }); + const windowEndsAt = retained!.lastUsedAt.getTime() + WINDOW_MS; + await note(`${label}: run ended, lease retained`, providerLeaseId, { releasedAt: iso(retained!.lastUsedAt.getTime()), + windowEndsAt: iso(windowEndsAt) }); + return { runId, providerLeaseId, windowEndsAt }; + } + /** The production cleanup worker on a real coordinator, as the server + * registers it at boot. Returns a stop function. */ + async function startCleanupWorker(workerDb: ReturnType) { + const heartbeat = heartbeatService(workerDb, { environmentRuntime: environmentRuntimeService(workerDb, { pluginWorkerManager: manager() }) }); + const coordinator = createDeliveryWorkCoordinator({ owner: workerDb, canRun: () => true, onError: error => { console.error("cleanup worker error", (error as Error).name); } }); + const sweeps: string[] = []; + const worker = registerEnvironmentCleanupWork(workerDb, coordinator, { + sweepPending: async backoffMs => { sweeps.push(new Date().toISOString()); await heartbeat.sweepPendingCleanupLeases({ backoffMs }); }, + canRun: () => true, + minimumSpacingMs: 1_000, + }); + await worker.ready; + return { sweeps, nextWakeAt: () => coordinator.nextWakeAt(), stop: async () => { worker.stop(); await coordinator.stop(); } }; + } + async function waitForState(providerLeaseId: string, state: string, deadlineMs: number) { + while (Date.now() < deadlineMs) { + const observed = await provider(providerLeaseId); + if (observed.state === state) return Date.now(); + await sleep(3_000); + } + return null; + } + + beforeAll(async () => { + if (!process.env.DAYTONA_API_KEY) throw new Error("DAYTONA_API_KEY required"); + for (const url of [pluginUrl, manifestUrl]) { + await access(new URL(url)).catch(() => { + throw new Error("Build the Daytona provider first: pnpm --dir packages/plugins/sandbox-providers/daytona install && pnpm --dir packages/plugins/sandbox-providers/daytona build"); + }); + } + temporary = await startEmbeddedPostgresTestDatabase("live-warm-expiry-"); + db = createDb(temporary.connectionString); + plugin = (await import(pluginUrl)).default.definition; + manifest = (await import(manifestUrl)).default; + const require = createRequire(new URL("../../../packages/plugins/sandbox-providers/daytona/package.json", import.meta.url)); + const { Daytona } = require("@daytonaio/sdk"); + daytona = new Daytona({ apiKey: process.env.DAYTONA_API_KEY }); + await db.insert(companies).values({ id: ids.company, name: "Disposable warm expiry", issuePrefix: "LWE" }); + await db.insert(agents).values({ id: ids.agent, companyId: ids.company, name: "No provider turn", adapterType: "paperclip_runner" }); + await db.insert(plugins).values({ id: ids.plugin, pluginKey: manifest.id, packageName: "@paperclipai/plugin-daytona", + version: manifest.version, apiVersion: 1, categories: ["automation"], manifestJson: manifest, status: "ready", installOrder: 1 }); + await db.insert(environments).values({ id: ids.environment, name: `Warm expiry ${ids.environment.slice(0, 8)}`, driver: "sandbox", config }); + environment = (await db.select().from(environments).where(eq(environments.id, ids.environment)))[0] as unknown as Environment; + await db.insert(issues).values({ id: ids.issue, companyId: ids.company, title: "Warm sandbox expiry", assigneeAgentId: ids.agent }); + }, 120_000); + + afterAll(async () => { + const leftBehind: string[] = []; + for (const id of sandboxIds) { + try { + const sandbox = await daytona.get(id); + if (sandbox.labels?.["paperclip-company-id"] !== ids.company || sandbox.labels?.["paperclip-environment-id"] !== ids.environment) { + throw new Error("Fixture cleanup ownership mismatch"); + } + await sandbox.delete(60); + timeline.push({ at: new Date().toISOString(), step: "cleanup: sandbox deleted", providerLeaseId: id }); + } catch (error) { + leftBehind.push(id); + timeline.push({ at: new Date().toISOString(), step: "cleanup failed", providerLeaseId: id, error: (error as Error).message }); + } + } + if (process.env.PAPERCLIP_LIVE_WARM_EXPIRY_EVIDENCE) { + await writeFile(process.env.PAPERCLIP_LIVE_WARM_EXPIRY_EVIDENCE, JSON.stringify({ windowMs: WINDOW_MS, timeline, methods }, null, 2)); + } + await temporary?.cleanup(); + if (leftBehind.length) { + // The stop path disables provider auto-delete, so a sandbox left here + // would never be reaped: fail loudly with the exact IDs to delete. + console.error(`Live fixture sandboxes left behind: ${leftBehind.join(", ")}`); + throw new Error(`Live fixture cleanup failed for ${leftBehind.length} sandbox(es): ${leftBehind.join(", ")}`); + } + }, 180_000); + + it("keeps a warm sandbox inside its window, stops it after, resumes it, and stops it again after a restart", async () => { + const worker = await startCleanupWorker(db); + try { + // Turn 1. + const first = await warmTurn("turn 1"); + expect(await nextEnvironmentCleanupAt(db)).toBe(first.windowEndsAt); + // Inside the window: the worker sleeps until the deadline; the sandbox stays started. + await sleep(first.windowEndsAt - Date.now() - 20_000); + expect((await note("turn 1: 20s before window end", first.providerLeaseId)).state).toBe("started"); + // Past the window: the worker wakes by itself and stops the exact sandbox. + const stoppedAt = await waitForState(first.providerLeaseId, "stopped", first.windowEndsAt + 90_000); + const afterStop = await note("turn 1: stopped by cleanup worker", first.providerLeaseId, + { secondsAfterWindowEnd: stoppedAt === null ? null : Math.round((stoppedAt - first.windowEndsAt) / 1000) }); + expect(stoppedAt).not.toBeNull(); + expect(stoppedAt!).toBeGreaterThanOrEqual(first.windowEndsAt); + expect(afterStop.autoDeleteInterval).toBe(-1); + const [kept] = (await leaseRows()).filter(row => row.heartbeatRunId === first.runId); + expect(kept).toMatchObject({ status: "released", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "stopped" } } }); + + // Turn 2 on the same task resumes the same sandbox. + const second = await warmTurn("turn 2"); + expect(second.providerLeaseId).toBe(first.providerLeaseId); + expect((await provider(second.providerLeaseId)).state).toBe("started"); + } finally { + await worker.stop(); + } + + // Restart mid-window: a fresh database client, a freshly loaded plugin + // module, and a new cleanup worker, as at server boot. + const [retained] = (await leaseRows()).filter(row => row.status === "retained"); + const windowEndsAt = retained!.lastUsedAt.getTime() + WINDOW_MS; + await sleep(Math.min(30_000, windowEndsAt - Date.now() - 30_000)); + plugin = (await import(`${pluginUrl}?restart=${randomUUID()}`)).default.definition; + const restartedDb = createDb(temporary.connectionString); + timeline.push({ at: new Date().toISOString(), step: "restart: new process state, cleanup worker boots", windowEndsAt: iso(windowEndsAt) }); + const restarted = await startCleanupWorker(restartedDb); + try { + expect((await note("restart: boot pass inside window", retained!.providerLeaseId!)).state).toBe("started"); + const stoppedAt = await waitForState(retained!.providerLeaseId!, "stopped", windowEndsAt + 90_000); + await note("restart: stopped by rebooted cleanup worker", retained!.providerLeaseId!, + { secondsAfterWindowEnd: stoppedAt === null ? null : Math.round((stoppedAt - windowEndsAt) / 1000) }); + expect(stoppedAt).not.toBeNull(); + expect(stoppedAt!).toBeGreaterThanOrEqual(windowEndsAt); + } finally { + await restarted.stop(); + } + }, 15 * 60_000); +});