import { JsonRpcCallError } from "@paperclipai/plugin-sdk"; import { collectRunFailureDiagnostics } from "../services/run-failure-diagnostics.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; import { createHash, randomUUID } from "node:crypto"; import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; import { buildSshEnvLabFixtureConfig, getSshEnvLabSupport, startSshEnvLabFixture, stopSshEnvLabFixture, } from "@paperclipai/adapter-utils/ssh"; import { activityLog, costEvents, agents, builtInManagedResources, companies, companySecretVersions, companySecrets, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issues, plugins, projects, } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; import { resolveEnvironmentDriverConfigForRuntime } from "../services/environment-config.ts"; import { SANDBOX_CAPABILITY_KEYS, environmentRuntimeService, findReusableSandboxLeaseId, SandboxOrphanCleanupWriteError, } from "../services/environment-runtime.ts"; import * as sandboxProviderRuntime from "../services/sandbox-provider-runtime.ts"; import * as environmentsModule from "../services/environments.ts"; import { logger } from "../middleware/logger.ts"; import { resolveRunnerEnvironmentForRun } from "../services/runner-environment-lifecycle.js"; import { environmentService } from "../services/environments.ts"; import { remoteExecutionHasStopped } from "../services/remote-execution-termination.ts"; import { heartbeatService } from "../services/heartbeat.ts"; import { secretService } from "../services/secrets.ts"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.ts"; import { getActiveStepContext, runWithRuntimeParent, type StartupSpanContext, } from "@paperclipai/adapter-utils/acpx-engine/startup-timing"; import { traceparentFromContextToken } from "../instrumentation.ts"; import { ROOT_CONTEXT, trace } from "@opentelemetry/api"; import { buildNativeHarnessBackupManifest } from "../services/native-runtime/native-session-executor.ts"; import { createNativeHarnessBackupStamp } from "../services/native-runtime/native-harness-backup-stamp.ts"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; const sshFixtureSupport = await getSshEnvLabSupport(); if (!embeddedPostgresSupport.supported) { console.warn( `Skipping embedded Postgres environment runtime tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, ); } function stableStringify(value: unknown): string { if (Array.isArray(value)) { return `[${value.map((entry) => stableStringify(entry)).join(",")}]`; } if (value && typeof value === "object") { const record = value as Record; return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`).join(",")}}`; } return JSON.stringify(value) ?? "null"; } function reusableRuntimeFingerprint(input: { provider: string; adapterType: string | null; config: Record; }) { return createHash("sha256").update(stableStringify(input)).digest("hex"); } describe("findReusableSandboxLeaseId", () => { it("matches reusable plugin-backed sandbox leases by provider", () => { const selected = findReusableSandboxLeaseId({ config: { provider: "fake-plugin", image: "template-b", timeoutMs: 300000, reuseLease: true, }, leases: [ { providerLeaseId: "sandbox-template-a", metadata: { provider: "fake-plugin", image: "template-a", timeoutMs: 300000, reuseLease: true, }, }, { providerLeaseId: "sandbox-template-b", metadata: { provider: "fake-plugin", image: "template-b", timeoutMs: 300000, reuseLease: true, }, }, ], }); expect(selected).toBe("sandbox-template-b"); }); it("ignores host-only log streaming when matching a reusable plugin lease", () => { const selected = findReusableSandboxLeaseId({ config: { provider: "fake-plugin", image: "template-b", timeoutMs: 300000, reuseLease: true, streamRunLogs: true, runnerLifecycleMode: "warm", target: null, }, leases: [ { providerLeaseId: "sandbox-template-b", metadata: { provider: "fake-plugin", image: "template-b", timeoutMs: 300000, reuseLease: true, runnerLifecycleMode: "warm", target: "us", }, }, ], }); expect(selected).toBe("sandbox-template-b"); }); it("requires image identity for reusable fake sandbox leases", () => { const selected = findReusableSandboxLeaseId({ config: { provider: "fake", image: "ubuntu:24.04", reuseLease: true, }, leases: [ { providerLeaseId: "sandbox-image-a", metadata: { provider: "fake", image: "debian:12", reuseLease: true, }, }, { providerLeaseId: "sandbox-image-b", metadata: { provider: "fake", image: "ubuntu:24.04", reuseLease: true, }, }, ], }); expect(selected).toBe("sandbox-image-b"); }); }); describeEmbeddedPostgres("environmentRuntimeService", () => { let stopDb: (() => Promise) | null = null; let db!: ReturnType; let runtime!: ReturnType; const fixtureRoots: string[] = []; // Give each test its own orphan-cleanup spool directory, so a spool file one // test writes never leaks into another test's flush. The default runtime reads // this env override when a test does not pass an explicit spool directory. let orphanCleanupSpoolDir: string | null = null; const previousOrphanCleanupSpoolDir = process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR; beforeAll(async () => { const started = await startEmbeddedPostgresTestDatabase("environment-runtime"); stopDb = started.stop; db = createDb(started.connectionString); runtime = environmentRuntimeService(db); }); beforeEach(async () => { orphanCleanupSpoolDir = await mkdtemp(path.join(os.tmpdir(), "orphan-cleanup-spool-")); process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR = orphanCleanupSpoolDir; }); afterEach(async () => { if (orphanCleanupSpoolDir) { await rm(orphanCleanupSpoolDir, { recursive: true, force: true }).catch(() => undefined); orphanCleanupSpoolDir = null; } if (previousOrphanCleanupSpoolDir === undefined) { delete process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR; } else { process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR = previousOrphanCleanupSpoolDir; } while (fixtureRoots.length > 0) { const root = fixtureRoots.pop(); if (!root) continue; await stopSshEnvLabFixture(path.join(root, "state.json")).catch(() => undefined); await rm(root, { recursive: true, force: true }).catch(() => undefined); } await db.delete(activityLog); await db.delete(environmentLeases); await db.delete(issues); await db.delete(costEvents); await db.delete(heartbeatRuns); await db.delete(agents); await db.delete(environments); await db.delete(executionWorkspaces); await db.delete(plugins); await db.delete(companySecretVersions); await db.delete(companySecrets); await db.delete(projects); await db.delete(companies); }); afterAll(async () => { await stopDb?.(); }); function insertSandboxProviderPlugin(pluginId: string, input: { pluginKey?: string; packageName?: string; displayName?: string; description?: string; author?: string; environmentDrivers?: Array>; status?: string; installOrder?: number; } = {}) { const pluginKey = input.pluginKey ?? "paperclip.fake-plugin-sandbox-provider"; return db.insert(plugins).values({ id: pluginId, pluginKey, packageName: input.packageName ?? "@paperclipai/plugin-fake-sandbox", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: pluginKey, apiVersion: 1, version: "1.0.0", displayName: input.displayName ?? "Fake Plugin Sandbox Provider", description: input.description ?? "Test fake plugin provider", author: input.author ?? "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: input.environmentDrivers ?? [{ driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", configSchema: { type: "object" } }], }, status: input.status ?? "ready", installOrder: input.installOrder ?? 1, updatedAt: new Date(), } as any); } function insertForeignCompanyEnvironmentBinding(otherCompanyId: string, environment: { id: string }) { return db.insert(builtInManagedResources).values({ companyId: otherCompanyId, bundleKey: "managed-environment", resourceKind: "environment", resourceKey: "managed-sandbox", resourceId: environment.id, stockVersion: "1", stockHash: "hash", }); } function insertReusableProject(projectId: string, companyId: string) { return db.insert(projects).values({ id: projectId, companyId, name: `Workspace ${projectId.slice(0, 8)}`, status: "active", createdAt: new Date(), updatedAt: new Date(), }); } function createFreshPluginLeaseWorker(pluginId: string) { return { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "fresh-plugin-lease", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, remoteCwd: "/workspace", }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; } function insertReusableExecutionWorkspace(executionWorkspaceId: string, companyId: string, projectId: string) { return db.insert(executionWorkspaces).values({ id: executionWorkspaceId, companyId, projectId, mode: "shared_workspace", strategyType: "project_primary", name: "Reusable workspace", status: "active", providerType: "local_fs", createdAt: new Date(), updatedAt: new Date(), }); } function insertStaleReusablePluginLease(companyId: string, environment: { id: string }, executionWorkspaceId: string, runId: string, agentId: string, pluginId: string, providerConfig: Record) { return environmentService(db).acquireLease({ companyId, environmentId: environment.id, executionWorkspaceId, heartbeatRunId: runId, leasePolicy: "reuse_by_environment", provider: "fake-plugin", providerLeaseId: "stale-plugin-lease", metadata: { agentId, driver: "sandbox", pluginId, pluginKey: "acme.fake-sandbox-provider", sandboxProviderPlugin: true, provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, reusableSandboxLease: { version: 1, companyId, environmentId: environment.id, executionWorkspaceId, agentId, adapterType: null, provider: "fake-plugin", runtimeFingerprint: reusableRuntimeFingerprint({ provider: "fake-plugin", adapterType: null, config: providerConfig, }), }, }, }); } function createReferencedWorkspaceRealizationRequest(companyId: string, environment: { id: string }, runId: string) { return { version: 1, adapterType: "codex_local", companyId, environmentId: environment.id, executionWorkspaceId: null, issueId: null, heartbeatRunId: runId, requestedMode: "ephemeral", source: { kind: "project_primary", localPath: "/tmp/anchor", projectId: "anchor-project", projectWorkspaceId: "anchor-workspace", repoUrl: null, repoRef: null, strategy: "project_primary", branchName: null, worktreePath: null, }, additionalSources: [ { localPath: "/tmp/referenced-project", projectId: "referenced-project-1", projectWorkspaceId: "referenced-workspace-1", repoUrl: null, repoRef: null, }, ], }; } function insertOldReusablePluginLease(companyId: string, environment: { id: string }, executionWorkspaceId: string, runId: string, agentId: string, providerConfig: Record) { return environmentService(db).acquireLease({ companyId, environmentId: environment.id, executionWorkspaceId, heartbeatRunId: runId, leasePolicy: "reuse_by_environment", provider: "fake-plugin", providerLeaseId: "old-plugin-lease", metadata: { agentId, provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, reusableSandboxLease: { version: 1, companyId, environmentId: environment.id, executionWorkspaceId, agentId, adapterType: null, provider: "fake-plugin", runtimeFingerprint: reusableRuntimeFingerprint({ provider: "fake-plugin", adapterType: null, config: providerConfig, }), }, }, }); } function insertRunningManualRun(nextRunId: string, companyId: string, agentId: string) { return db.insert(heartbeatRuns).values({ id: nextRunId, companyId, agentId, invocationSource: "manual", status: "running", createdAt: new Date(), updatedAt: new Date(), }); } async function seedEnvironment(input: { driver?: string; name?: string; status?: "active" | "disabled"; config?: Record; } = {}) { const companyId = randomUUID(); const agentId = randomUUID(); const environmentId = randomUUID(); const runId = randomUUID(); const driver = input.driver ?? "local"; const environmentName = input.name ?? `${driver}-${environmentId.slice(0, 8)}`; let config = input.config ?? {}; await db.insert(companies).values({ id: companyId, name: "Acme", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(agents).values({ id: agentId, companyId, name: "CodexCoder", role: "engineer", status: "active", adapterType: "codex_local", adapterConfig: {}, runtimeConfig: {}, permissions: {}, createdAt: new Date(), updatedAt: new Date(), }); if (typeof config.privateKey === "string" && config.privateKey.length > 0) { const secret = await secretService(db).create(companyId, { name: `environment-runtime-private-key-${randomUUID()}`, provider: "local_encrypted", value: config.privateKey, }); await secretService(db).createBinding({ companyId, secretId: secret.id, targetType: "environment", targetId: environmentId, configPath: "privateKeySecretRef", }); config = { ...config, privateKey: null, privateKeySecretRef: { type: "secret_ref", secretId: secret.id, version: "latest", }, }; } const existingLocalEnvironment = driver === "local" ? await db .select() .from(environments) .where(eq(environments.driver, "local")) .then((rows) => rows[0] ?? null) : null; const environmentRecord = existingLocalEnvironment ?? { id: environmentId, name: environmentName, description: null, driver, status: input.status ?? "active", config, metadata: null, createdAt: new Date(), updatedAt: new Date(), }; if (!existingLocalEnvironment) { await db.insert(environments).values({ id: environmentRecord.id, name: environmentRecord.name, driver: environmentRecord.driver, status: environmentRecord.status, config: environmentRecord.config, createdAt: environmentRecord.createdAt, updatedAt: environmentRecord.updatedAt, }); } await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "manual", status: "running", createdAt: new Date(), updatedAt: new Date(), }); return { companyId, agentId, environment: { id: environmentRecord.id, companyId, name: environmentRecord.name, description: environmentRecord.description, driver: environmentRecord.driver, status: environmentRecord.status, config: environmentRecord.config, metadata: environmentRecord.metadata, createdAt: environmentRecord.createdAt, updatedAt: environmentRecord.updatedAt, } as const, runId, }; } async function seedReusablePluginSandboxLease(adapterType: string | null = null) { const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, }; const environment = { ...baseEnvironment, name: "Reusable Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "acme.reusable-sandbox-provider", packageName: "@acme/reusable-sandbox-provider", displayName: "Reusable Sandbox Provider", description: "Test provider with reusable lease support", environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", supportsReusableLeases: true, configSchema: { type: "object", properties: { image: { type: "string" }, timeoutMs: { type: "number" }, reuseLease: { type: "boolean" }, }, }, }, ], }); const executionWorkspaceId = randomUUID(); const projectId = randomUUID(); await db.insert(projects).values({ id: projectId, companyId, name: `Workspace ${projectId.slice(0, 8)}`, status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(executionWorkspaces).values({ id: executionWorkspaceId, companyId, projectId, mode: "shared_workspace", strategyType: "project_primary", name: "Reusable workspace", status: "active", providerType: "local_fs", createdAt: new Date(), updatedAt: new Date(), }); const reusableLease = await environmentService(db).acquireLease({ companyId, environmentId: environment.id, executionWorkspaceId, heartbeatRunId: runId, leasePolicy: "reuse_by_environment", provider: "fake-plugin", providerLeaseId: "reusable-plugin-lease", metadata: { agentId, driver: "sandbox", pluginId, pluginKey: "acme.reusable-sandbox-provider", sandboxProviderPlugin: true, provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, reusableSandboxLease: { version: 1, companyId, environmentId: environment.id, executionWorkspaceId, agentId, adapterType, provider: "fake-plugin", runtimeFingerprint: reusableRuntimeFingerprint({ provider: "fake-plugin", adapterType, config: providerConfig, }), }, }, }); return { pluginId, companyId, agentId, environment, runId, executionWorkspaceId, reusableLease }; } it("acquires and resumes a reusable lease after a task changes from per-turn to warm", async () => { const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); await environmentService(db).releaseLease(seeded.reusableLease.id, "expired"); const environment = { ...seeded.environment, config: { ...seeded.environment.config, reuseLease: false } }; await environmentService(db).update(environment.id, { config: environment.config }); const issueId = randomUUID(); await db.insert(issues).values({ id: issueId, companyId: seeded.companyId, title: "Lifecycle switch", status: "in_progress", assigneeAgentId: seeded.agentId }); let acquisitions = 0; let warmProviderLeaseId = ""; const call = vi.fn(async (_pluginId: string, method: string, input: any) => { if (method === "environmentAcquireLease") return { providerLeaseId: `lifecycle-${++acquisitions}`, metadata: { remoteCwd: "/workspace" }, }; if (method === "environmentResumeLease") return { providerLeaseId: warmProviderLeaseId, metadata: { remoteCwd: "/workspace" }, }; if (method === "environmentDestroyLease" || method === "environmentStopLease" || method === "environmentReleaseLease") return { providerLeaseId: input.providerLeaseId, state: method === "environmentDestroyLease" ? "destroyed" : "stopped", }; throw new Error(`Unexpected lifecycle method: ${method}`); }); const runtime = environmentRuntimeService(db, { pluginWorkerManager: { isRunning: () => true, call, getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentStopLease", "environmentDestroyLease"] }), } as unknown as PluginWorkerManager }); const input = { companyId: seeded.companyId, agentId: seeded.agentId, issueId, adapterType: "paperclip_runner", persistedExecutionWorkspace: { id: seeded.executionWorkspaceId, mode: "shared_workspace" as const }, }; const cold = await runtime.acquireRunLease({ ...input, environment, heartbeatRunId: seeded.runId }); expect(cold.lease.leasePolicy).toBe("ephemeral"); await runtime.releaseRunLeases(seeded.runId, "released"); const warmEnvironment = resolveRunnerEnvironmentForRun(environment, "paperclip_runner", { lifecycleMode: "warm" }); const warmRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: warmRunId, companyId: seeded.companyId, agentId: seeded.agentId, status: "running" }); const warm = await runtime.acquireRunLease({ ...input, environment: warmEnvironment, heartbeatRunId: warmRunId }); warmProviderLeaseId = warm.lease.providerLeaseId!; expect(warm.lease.leasePolicy).toBe("reuse_by_environment"); expect(warm.lease.executionWorkspaceId).toBe(seeded.executionWorkspaceId); expect((await runtime.resolveCapabilities({ environment: warmEnvironment, lease: warm.lease })).reusableLeases).toBe(true); await runtime.releaseRunLeases(warmRunId, "released", undefined, "stop_and_retain"); const nextRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: nextRunId, companyId: seeded.companyId, agentId: seeded.agentId, status: "running" }); const next = await runtime.acquireRunLease({ ...input, environment: warmEnvironment, heartbeatRunId: nextRunId }); expect(next.lease.providerLeaseId).toBe(warmProviderLeaseId); expect(acquisitions).toBe(2); expect((await environmentService(db).getById(environment.id))?.config.reuseLease).toBe(false); }); it.each(["stopped", "missing", "wrong-lease", "error"])( "startup cancellation is run-scoped and needs a matching receipt: %s", async (outcome) => { const { pluginId, companyId, runId, reusableLease, environment } = await seedReusablePluginSandboxLease(); const other = await environmentService(db).acquireLease({ companyId, environmentId: environment.id, heartbeatRunId: null, leasePolicy: "ephemeral", provider: "fake-plugin", providerLeaseId: "other-sandbox", }); const call = vi.fn(async () => { if (outcome === "error") throw new Error("provider unavailable"); if (outcome === "missing") return undefined; return { providerLeaseId: outcome === "wrong-lease" ? "other-sandbox" : reusableLease.providerLeaseId, state: "stopped" }; }); const workerManager = { isRunning: () => true, call, getWorker: () => ({ supportedMethods: ["environmentStopLease"] }), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await runtimeWithPlugin.releaseRunLeases(runId, "released", undefined, "stop_and_retain", true); expect(call).toHaveBeenCalledWith(pluginId, "environmentStopLease", expect.objectContaining({ companyId, providerLeaseId: reusableLease.providerLeaseId, cancelActiveWork: true, }), expect.any(Number)); expect(call).toHaveBeenCalledOnce(); expect(await remoteExecutionHasStopped(db, companyId, runId)).toBe(outcome === "stopped"); await expect(environmentService(db).getLeaseById(other.id)).resolves.toMatchObject({ status: "active" }); }, ); it.each(["release_only", "stopped", "stop_failed", "unconfirmed", "restart", "competing_owner", "missing_pin", "late_receipt"])( "never dispatches destructive release for an untagged stop-and-retain request: %s", async outcome => { const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); const lease = seeded.reusableLease; await db.update(environmentLeases).set({ leasePolicy: "ephemeral", metadata: { ...lease.metadata, reuseLease: false, ...(outcome === "missing_pin" ? { pluginId: undefined } : {}), } }).where(eq(environmentLeases.id, lease.id)); await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId)); let supported = !["release_only", "restart"].includes(outcome); let calls = 0; const call = vi.fn(async (_id: string, method: string) => { if (method !== "environmentStopLease") return { providerLeaseId: lease.providerLeaseId, state: "destroyed" }; if (++calls === 1 && outcome === "stop_failed") throw new Error("injected stop failure"); if (calls === 1 && outcome === "unconfirmed") return undefined; if (outcome === "late_receipt") await db.update(environmentLeases).set({ status: "active", heartbeatRunId: null, cleanupStatus: null, metadata: { newOwner: true }, }).where(eq(environmentLeases.id, lease.id)); return { providerLeaseId: lease.providerLeaseId, state: "stopped" }; }); const runtime = () => environmentRuntimeService(db, { pluginWorkerManager: { isRunning: () => true, call, getWorker: () => ({ supportedMethods: ["environmentReleaseLease", "environmentDestroyLease", ...(supported ? ["environmentStopLease"] : [])] }), } as unknown as PluginWorkerManager }); if (outcome === "competing_owner") await environmentService(db).acquireLease({ companyId: seeded.companyId, environmentId: seeded.environment.id, heartbeatRunId: null, leasePolicy: "ephemeral", provider: lease.provider, providerLeaseId: lease.providerLeaseId, }); await runtime().releaseRunLeases(seeded.runId, "failed", undefined, "stop_and_retain"); expect(call.mock.calls.every(entry => entry[1] === "environmentStopLease")).toBe(true); if (outcome === "late_receipt") { expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "active", heartbeatRunId: null, metadata: { newOwner: true } }); return; } if (outcome === "missing_pin") { expect(call).not.toHaveBeenCalled(); await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 }); expect(call).not.toHaveBeenCalled(); expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "pending_cleanup" }); return; } if (["release_only", "restart", "competing_owner"].includes(outcome)) { expect(call).not.toHaveBeenCalled(); expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "pending_cleanup", metadata: { sandboxStopAndRetain: { pluginId: seeded.pluginId, runId: seeded.runId, providerLeaseId: lease.providerLeaseId }, } }); await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 }); expect(call).not.toHaveBeenCalled(); if (outcome === "competing_owner") return; supported = true; } if (["release_only", "stop_failed", "unconfirmed", "restart"].includes(outcome)) { await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 }); } expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "released", cleanupStatus: "success", failureReason: null, metadata: { remoteExecutionTermination: { state: "stopped", providerLeaseId: lease.providerLeaseId }, sandboxStopAndRetainReceipt: { method: "environmentStopLease", runId: seeded.runId, pluginId: seeded.pluginId } } }); expect(call.mock.calls.every(entry => entry[1] === "environmentStopLease")).toBe(true); }, ); it("reports an unwired manager separately from a stopped sandbox worker", async () => { const { companyId, environment, runId } = await seedReusablePluginSandboxLease(); const input = { companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }; const runtimeWithoutManager = environmentRuntimeService(db); await expect(runtimeWithoutManager.acquireRunLease(input)) .rejects.toThrow("sandbox plugin workers are unavailable in this server process"); const offlineManager = { isRunning: () => false, call: vi.fn() } as unknown as PluginWorkerManager; const runtimeWithStoppedWorker = environmentRuntimeService(db, { pluginWorkerManager: offlineManager, pluginWorkerReadyTimeoutMs: 0, }); await expect(runtimeWithStoppedWorker.acquireRunLease(input)) .rejects.toThrow("its worker is not running"); expect(offlineManager.call).not.toHaveBeenCalled(); }); it("retains a successful reusable sandbox lease without stopping the provider resource", async () => { const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease(); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { throw new Error(`Unexpected plugin method while retaining warm lease: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const released = await runtimeWithPlugin.releaseRunLeases( runId, "released", undefined, "keep_running", ); expect(released).toHaveLength(1); expect(released[0]?.lease).toMatchObject({ id: reusableLease.id, status: "retained", cleanupStatus: "success", }); await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ status: "retained", cleanupStatus: "success", }); expect(workerManager.call).not.toHaveBeenCalled(); }); it("stops a reusable sandbox and keeps its lease resumable", async () => { const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease(); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentStopLease") return { providerLeaseId: reusableLease.providerLeaseId, state: "stopped" }; throw new Error(`Unexpected plugin method while stopping lease: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentStopLease", "environmentDestroyLease", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const released = await runtimeWithPlugin.releaseRunLeases( runId, "failed", undefined, "stop_and_retain", ); expect(released[0]?.lease).toMatchObject({ id: reusableLease.id, status: "released", }); expect(workerManager.call).toHaveBeenCalledWith( pluginId, "environmentStopLease", expect.objectContaining({ providerLeaseId: reusableLease.providerLeaseId }), expect.any(Number), ); }); it.each(["same task", "different task", "different agent", "no task"])( "scopes projectless reusable leases to their task and agent: %s", async (scenario) => { const seeded = await seedReusablePluginSandboxLease(); await environmentService(db).releaseLease(seeded.reusableLease.id, "expired"); const issueId = randomUUID(); const otherIssueId = randomUUID(); await db.insert(issues).values([issueId, otherIssueId].map(id => ({ id, companyId: seeded.companyId, title: "Projectless chat", status: "in_progress", assigneeAgentId: seeded.agentId, }))); const nextAgentId = scenario === "different agent" ? randomUUID() : seeded.agentId; if (nextAgentId !== seeded.agentId) await db.insert(agents).values({ id: nextAgentId, companyId: seeded.companyId, name: "Other agent", adapterType: "paperclip_runner", }); let acquisitions = 0; const call = vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") return { providerLeaseId: `projectless-${++acquisitions}`, metadata: { remoteCwd: "/workspace" }, }; if (method === "environmentResumeLease") return { providerLeaseId: "projectless-1", metadata: { remoteCwd: "/workspace" }, }; if (method === "environmentStopLease") return { providerLeaseId: "projectless-1", state: "stopped" }; throw new Error(`Unexpected projectless lease method: ${method}`); }); const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: { isRunning: () => true, call, getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentStopLease", "environmentDestroyLease"] }), } as unknown as PluginWorkerManager }); const first = await runtimeWithPlugin.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment, issueId, agentId: seeded.agentId, heartbeatRunId: seeded.runId, persistedExecutionWorkspace: null, }); expect(first.lease.metadata?.reusableSandboxLease).toMatchObject({ projectlessIssueId: issueId, executionWorkspaceId: null, agentId: seeded.agentId, }); await runtimeWithPlugin.releaseRunLeases(seeded.runId, "released", undefined, "stop_and_retain"); const nextRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: nextRunId, companyId: seeded.companyId, agentId: nextAgentId, status: "running", }); if (scenario === "different task" || scenario === "different agent") { await expect(environmentService(db).acquireLease({ companyId: seeded.companyId, environmentId: seeded.environment.id, issueId: scenario === "different task" ? otherIssueId : issueId, heartbeatRunId: nextRunId, executionWorkspaceId: null, provider: "fake-plugin", providerLeaseId: first.lease.providerLeaseId, leasePolicy: "reuse_by_environment", replacesReusableLeaseId: first.lease.id, metadata: { agentId: nextAgentId }, })).rejects.toMatchObject({ status: 409 }); } const next = await runtimeWithPlugin.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment, issueId: scenario === "no task" ? null : scenario === "different task" ? otherIssueId : issueId, agentId: nextAgentId, heartbeatRunId: nextRunId, persistedExecutionWorkspace: null, }); const shouldResume = scenario === "same task"; expect(next.lease.providerLeaseId).toBe(shouldResume ? "projectless-1" : "projectless-2"); expect(acquisitions).toBe(shouldResume ? 1 : 2); expect(call.mock.calls.filter((entry) => entry[1] === "environmentResumeLease")).toHaveLength(shouldResume ? 1 : 0); }); it.each(["pending", "inline", "foreign scope", "foreign environment", "foreign run", "malformed", "deleted environment", "forged error", "invalid diagnostic", "old worker"])("retains uncertain plugin creation for durable cleanup: %s", async (mode) => { const seeded = await seedReusablePluginSandboxLease(); await environmentService(db).releaseLease(seeded.reusableLease.id, "expired"); const attemptId = randomUUID(); const providerLeaseId = `paperclip-create-${attemptId}`; const cleanup = { providerLeaseId, attemptId, companyId: mode === "foreign scope" ? randomUUID() : seeded.companyId, environmentId: mode === "foreign environment" ? randomUUID() : seeded.environment.id, runId: mode === "foreign run" ? randomUUID() : seeded.runId, accountFingerprint: mode === "malformed" ? "invalid" : "a".repeat(64), labels: { "paperclip-provider": "fake-plugin" }, }; const acquisitionDiagnostic = { phase: "shell", elapsedMs: 300_000, budgetMs: 300_000 }; const data = { schema: "paperclip/environment-creation-cleanup/v1", cleanup, ...(mode === "old worker" ? {} : { acquisitionDiagnostic: mode === "invalid diagnostic" ? { ...acquisitionDiagnostic, phase: "private-command" } : acquisitionDiagnostic }), }; const failure = mode === "forged error" ? Object.assign(new Error("Provider creation cleanup required"), { data }) : new JsonRpcCallError({ code: -32002, message: "Provider creation cleanup required", data }); let available = mode === "inline"; const call = vi.fn(async (_id: string, method: string, params: Record) => { if (method === "environmentAcquireLease") throw failure; if (method === "environmentDestroyLease") { const pending = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, providerLeaseId)); expect(pending).toHaveLength(1); expect(pending[0].status).toBe("pending_cleanup"); expect(params).toMatchObject({ providerLeaseId, companyId: seeded.companyId, environmentId: seeded.environment.id, leaseMetadata: { failedCreateCleanup: cleanup } }); if (!available) throw new Error("provider not visible yet"); return { providerLeaseId, state: "destroyed" }; } throw new Error(`Unexpected method ${method}`); }); const worker = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }), call } as unknown as PluginWorkerManager; const candidate = environmentRuntimeService(db, { pluginWorkerManager: worker }); const acquisition = candidate.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment, issueId: null, heartbeatRunId: seeded.runId, persistedExecutionWorkspace: null }); const caught = await acquisition.catch(error => error); const execution = collectRunFailureDiagnostics({ id: seeded.runId, companyId: seeded.companyId, errorCode: "setup_failed", resultJson: null, } as typeof heartbeatRuns.$inferSelect, { error: caught, phase: "setup" }).execution; if (["pending", "inline", "deleted environment"].includes(mode)) { expect(execution).toMatchObject({ environmentAcquisitionPhase: "shell", environmentAcquisitionElapsedMs: 300_000, environmentAcquisitionBudgetMs: 300_000 }); } else expect(execution).not.toHaveProperty("environmentAcquisitionPhase"); if (mode === "inline") { await expect(acquisition).rejects.toMatchObject({ message: "Sandbox creation failed; allocated sandbox cleanup was confirmed.", cause: failure, }); } else { await expect(acquisition).rejects.toBe(failure); } const rows = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, providerLeaseId)); if (["foreign scope", "foreign environment", "foreign run", "malformed"].includes(mode)) { expect(rows).toHaveLength(0); expect(call.mock.calls.filter((c) => c[1] === "environmentDestroyLease")).toHaveLength(0); return; } expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ status: mode === "inline" ? "expired" : "pending_cleanup", cleanupStatus: mode === "inline" ? "success" : "failed", companyId: seeded.companyId, metadata: { pluginId: seeded.pluginId, sandboxProviderPlugin: true, failedCreateCleanup: cleanup } }); if (mode === "pending" || mode === "deleted environment") { available = true; if (mode === "deleted environment") { await db.update(environmentLeases).set({ environmentId: null }).where(eq(environmentLeases.id, rows[0].id)); } const restarted = environmentRuntimeService(db, { pluginWorkerManager: worker }); const persisted = await environmentService(db).getLeaseById(rows[0].id); await expect(restarted.retryPendingSandboxTeardown({ environment: null, lease: persisted! })).resolves.toEqual({ providerLeaseId, state: "destroyed" }); } }); it.each(["deleted receipt lost", "observation response lost", "journal write failed", "foreign observation"])( "journals observed creation identity before deletion and recovers: %s", async (fault) => { const seeded = await seedReusablePluginSandboxLease(); await environmentService(db).releaseLease(seeded.reusableLease.id, "expired"); const cleanup = { providerLeaseId: `paperclip-create-${randomUUID()}`, attemptId: randomUUID(), companyId: seeded.companyId, environmentId: seeded.environment.id, runId: seeded.runId, accountFingerprint: "a".repeat(64), labels: { "paperclip-provider": "fake-plugin" }, }; const observed = { ...cleanup, observedProviderLeaseId: "verified-provider-id" }; const failure = Object.assign(new Error("Creation uncertain"), { data: { schema: "paperclip/environment-creation-cleanup/v1", cleanup }, }); let injected = false; let deleted = false; const call = vi.fn(async (_id: string, method: string, params: Record) => { if (method === "environmentAcquireLease") throw failure; if (method !== "environmentDestroyLease") throw new Error(`Unexpected ${method}`); const evidence = (params.leaseMetadata as { failedCreateCleanup: typeof observed }).failedCreateCleanup; if (!evidence.observedProviderLeaseId) { expect(deleted).toBe(false); if (!injected && fault === "observation response lost") { injected = true; throw new Error("Lost observation response"); } if (!injected && fault === "foreign observation") { injected = true; throw Object.assign(new Error("Foreign observation"), { data: { schema: "paperclip/environment-creation-cleanup/v1", cleanup: { ...observed, companyId: "another-company" } }, }); } if (!injected && fault === "journal write failed") { injected = true; vi.spyOn(db, "update").mockImplementationOnce(() => { throw new Error("Database unavailable"); }); } throw Object.assign(new Error("Persist observed sandbox identity before cleanup"), { data: { schema: "paperclip/environment-creation-cleanup/v1", cleanup: observed }, }); } const rows = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, cleanup.providerLeaseId)); expect(rows).toHaveLength(1); expect(rows[0].metadata).toMatchObject({ failedCreateCleanup: observed }); if (!deleted) { deleted = true; if (fault === "deleted receipt lost") throw new Error("Deletion succeeded but reply lost"); } return { providerLeaseId: cleanup.providerLeaseId, state: "destroyed" }; }); const worker = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }), call } as unknown as PluginWorkerManager; const runtime = environmentRuntimeService(db, { pluginWorkerManager: worker }); await expect(runtime.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment, issueId: null, heartbeatRunId: seeded.runId, persistedExecutionWorkspace: null })).rejects.toBe(failure); const rows = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, cleanup.providerLeaseId)); expect(rows).toHaveLength(1); expect(rows[0].status).toBe("pending_cleanup"); expect(deleted).toBe(fault === "deleted receipt lost"); const restarted = environmentRuntimeService(db, { pluginWorkerManager: worker }); await expect(restarted.retryPendingSandboxTeardown({ environment: null, lease: rows[0]! })).resolves.toEqual({ providerLeaseId: cleanup.providerLeaseId, state: "destroyed" }); expect(deleted).toBe(true); }, ); it("resumes the same provider lease on the next per-turn run", async () => { const seeded = await seedReusablePluginSandboxLease(); const workerManager = { isRunning: vi.fn((id: string) => id === seeded.pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-exact-resume", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, remoteCwd: "/workspace", }, }; } if (method === "environmentStopLease") return { providerLeaseId: "sandbox-exact-resume", state: "stopped" }; if (method === "environmentResumeLease") { return { providerLeaseId: "sandbox-exact-resume", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, remoteCwd: "/workspace", }, }; } throw new Error(`Unexpected plugin method during exact resume: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentStopLease", "environmentDestroyLease", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await environmentService(db).releaseLease(seeded.reusableLease.id, "expired"); const first = await runtimeWithPlugin.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment, issueId: null, agentId: seeded.agentId, heartbeatRunId: seeded.runId, persistedExecutionWorkspace: { id: seeded.executionWorkspaceId, mode: "shared_workspace", }, }); const workspaceSyncStamp = { schema: "paperclip.native-workspace-stamp/v1", workspaceId: seeded.executionWorkspaceId, providerLeaseId: "sandbox-exact-resume", remoteCwd: "/workspace", hostSha256: "a".repeat(64), finalizedRunId: seeded.runId, }; await environmentService(db).updateLeaseMetadata(first.lease.id, { ...(first.lease.metadata ?? {}), nativeWorkspaceSync: workspaceSyncStamp, }); await runtimeWithPlugin.releaseRunLeases( seeded.runId, "released", undefined, "stop_and_retain", ); const nextRunId = randomUUID(); await db.insert(heartbeatRuns).values({ id: nextRunId, companyId: seeded.companyId, agentId: seeded.agentId, invocationSource: "manual", status: "running", createdAt: new Date(), updatedAt: new Date(), }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment, issueId: null, agentId: seeded.agentId, heartbeatRunId: nextRunId, persistedExecutionWorkspace: { id: seeded.executionWorkspaceId, mode: "shared_workspace", }, }); expect(first.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "created" }); expect(acquired.lease.providerLeaseId).toBe("sandbox-exact-resume"); expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "resumed", }); expect(acquired.lease.metadata?.nativeWorkspaceSync).toEqual( workspaceSyncStamp, ); await expect( environmentService(db).getLeaseById(first.lease.id), ).resolves.toMatchObject({ status: "expired", cleanupStatus: "success", }); expect( workerManager.call.mock.calls.filter( (call) => call[1] === "environmentAcquireLease", ), ).toHaveLength(1); }); it("reacquires one provider lease row idempotently during same-run recovery", async () => { const seeded = await seedReusablePluginSandboxLease(); const workerManager = { isRunning: vi.fn((id: string) => id === seeded.pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentResumeLease") { return { providerLeaseId: seeded.reusableLease.providerLeaseId, metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, remoteCwd: "/workspace", }, }; } throw new Error( `Unexpected plugin method during same-run reacquisition: ${method}`, ); }), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager, }); const reacquired = await runtimeWithPlugin.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment, issueId: null, agentId: seeded.agentId, heartbeatRunId: seeded.runId, persistedExecutionWorkspace: { id: seeded.executionWorkspaceId, mode: "shared_workspace", }, }); expect(reacquired.lease).toMatchObject({ id: seeded.reusableLease.id, heartbeatRunId: seeded.runId, providerLeaseId: seeded.reusableLease.providerLeaseId, status: "active", cleanupStatus: null, }); const rows = await environmentService(db).listLeases(seeded.environment.id); expect( rows.filter( (lease) => lease.heartbeatRunId === seeded.runId && lease.providerLeaseId === seeded.reusableLease.providerLeaseId, ), ).toHaveLength(1); expect(workerManager.call).toHaveBeenCalledWith( seeded.pluginId, "environmentResumeLease", expect.objectContaining({ providerLeaseId: seeded.reusableLease.providerLeaseId, }), expect.any(Number), ); }); it("destroys a disposable paperclip_runner sandbox after the turn", async () => { const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease(); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentDestroyLease") return undefined; throw new Error(`Unexpected plugin method while destroying lease: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const released = await runtimeWithPlugin.releaseRunLeases( runId, "released", undefined, "destroy", ); expect(released[0]?.lease).toMatchObject({ id: reusableLease.id, status: "expired", }); expect(workerManager.call).toHaveBeenCalledWith( pluginId, "environmentDestroyLease", expect.objectContaining({ providerLeaseId: reusableLease.providerLeaseId }), expect.any(Number), ); }); it("does not destroy native sandbox state when its verified backup is missing", async () => { const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease(); await environmentService(db).updateLeaseMetadata(reusableLease.id, { ...(reusableLease.metadata ?? {}), reusableSandboxLease: { ...((reusableLease.metadata?.reusableSandboxLease as Record< string, unknown >) ?? {}), adapterType: "paperclip_runner", }, sandboxLeaseAcquisition: { outcome: "created" }, }); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { throw new Error(`Provider teardown must not run without a backup: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const errors: unknown[] = []; const released = await runtimeWithPlugin.releaseRunLeases( runId, "released", (_leaseId, error) => errors.push(error), "destroy", ); expect(released).toEqual([]); expect(errors).toHaveLength(1); expect(String(errors[0])).toContain("runner_harness_backup_unavailable"); expect(workerManager.call).not.toHaveBeenCalled(); await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ status: "active", }); }); it("acquires and releases a projectless local run lease through the runtime seam", async () => { const { companyId, environment, runId } = await seedEnvironment(); const acquired = await runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); expect(acquired.lease.status).toBe("active"); expect(acquired.lease.executionWorkspaceId).toBeNull(); expect(acquired.lease.metadata).toMatchObject({ driver: "local", executionWorkspaceMode: null, }); expect(acquired.leaseContext).toEqual({ executionWorkspaceId: null, executionWorkspaceMode: null, }); const released = await runtime.releaseRunLeases(runId); expect(released).toHaveLength(1); expect(released[0]?.environment.driver).toBe("local"); expect(released[0]?.lease.status).toBe("released"); const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.id, acquired.lease.id)); expect(rows[0]?.status).toBe("released"); }); it("rejects truly unsupported drivers before acquiring a lease", async () => { const { companyId, agentId, environment, runId } = await seedEnvironment({ driver: "ssh", name: "Fixture SSH", config: { host: "ssh.example.test", port: 22, username: "ssh-user", remoteWorkspacePath: "/srv/paperclip/workspace", privateKey: null, knownHosts: null, strictHostKeyChecking: true, }, }); const runtimeWithoutSsh = environmentRuntimeService(db, { drivers: [ { driver: "local", acquireRunLease: async () => { throw new Error("should not acquire"); }, releaseRunLease: async () => null, }, ], }); await expect( runtimeWithoutSsh.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }), ).rejects.toThrow('Environment driver "ssh" is not registered in the environment runtime yet.'); const rows = await db.select().from(environmentLeases); expect(rows).toHaveLength(0); }); it("acquires and releases an SSH run lease through the runtime seam", async () => { if (!sshFixtureSupport.supported) { console.warn( `Skipping SSH runtime fixture test: ${sshFixtureSupport.reason ?? "unsupported environment"}`, ); return; } const fixtureRoot = await mkdtemp(path.join(os.tmpdir(), "paperclip-environment-runtime-ssh-")); fixtureRoots.push(fixtureRoot); const statePath = path.join(fixtureRoot, "state.json"); const fixture = await startSshEnvLabFixture({ statePath }); const sshConfig = await buildSshEnvLabFixtureConfig(fixture); const { companyId, environment, runId } = await seedEnvironment({ driver: "ssh", name: "Fixture SSH", config: sshConfig, }); try { const acquired = await runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); expect(acquired.lease.status).toBe("active"); expect(acquired.lease.providerLeaseId).toContain(`ssh://${sshConfig.username}@${sshConfig.host}:${sshConfig.port}`); expect(acquired.lease.metadata).toMatchObject({ driver: "ssh", host: sshConfig.host, port: sshConfig.port, username: sshConfig.username, remoteWorkspacePath: sshConfig.remoteWorkspacePath, remoteCwd: sshConfig.remoteWorkspacePath, }); const released = await runtime.releaseRunLeases(runId); expect(released).toHaveLength(1); expect(released[0]?.environment.driver).toBe("ssh"); expect(released[0]?.lease.status).toBe("released"); } finally { } }); it("acquires and releases a fake sandbox run lease through the runtime seam", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Fake Sandbox", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: true, }, }); const acquired = await runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); expect(acquired.lease.status).toBe("active"); expect(acquired.lease.providerLeaseId).toMatch(new RegExp(`^sandbox://fake/${runId}/[0-9a-f-]{36}$`)); expect(acquired.lease.leasePolicy).toBe("ephemeral"); expect(acquired.lease.metadata).toMatchObject({ driver: "sandbox", provider: "fake", image: "ubuntu:24.04", reuseLease: true, }); const released = await runtime.releaseRunLeases(runId); expect(released).toHaveLength(1); expect(released[0]?.environment.driver).toBe("sandbox"); expect(released[0]?.lease.status).toBe("released"); }); it.each([false, true])("confirms a built-in fake stop without release or destroy fallback: cancel=%s", async cancel => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const acquired = await runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null }); const provider = sandboxProviderRuntime.requireSandboxProvider("fake"); const release = vi.spyOn(provider, "releaseLease"), destroy = vi.spyOn(provider, "destroyLease"); const released = await runtime.releaseRunLeases(runId, "released", undefined, "stop_and_retain", cancel); expect(released).toHaveLength(1); expect(released[0]?.lease).toMatchObject({ status: "released", cleanupStatus: "success", failureReason: null, metadata: { remoteExecutionTermination: { state: "stopped", providerLeaseId: acquired.lease.providerLeaseId }, sandboxStopAndRetainReceipt: { method: "builtin.stopLease", builtinProvider: "fake", runId } } }); expect(await remoteExecutionHasStopped(db, companyId, runId)).toBe(true); expect(release).not.toHaveBeenCalled(); expect(destroy).not.toHaveBeenCalled(); }); it.each(["stop_failed", "wrong_receipt", "missing_capability", "changed_binding"])( "preserves a built-in stop through restart without destructive fallback: %s", async outcome => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const acquired = await runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null }); const provider = sandboxProviderRuntime.requireSandboxProvider("fake"), originalStop = provider.stopLease!; const release = vi.spyOn(provider, "releaseLease"), destroy = vi.spyOn(provider, "destroyLease"); try { provider.stopLease = outcome === "missing_capability" ? undefined : vi.fn(async () => { if (outcome === "wrong_receipt") return { providerLeaseId: "sandbox://fake/other", state: "stopped" as const }; throw new Error("injected built-in stop failure"); }); await runtime.releaseRunLeases(runId, "released", undefined, "stop_and_retain", true); expect(await environmentService(db).getLeaseById(acquired.lease.id)).toMatchObject({ status: "pending_cleanup" }); expect(await remoteExecutionHasStopped(db, companyId, runId)).toBe(false); if (outcome === "missing_capability") { await heartbeatService(db, { environmentRuntime: environmentRuntimeService(db) }).sweepPendingCleanupLeases({ backoffMs: 0 }); expect(await environmentService(db).getLeaseById(acquired.lease.id)).toMatchObject({ status: "pending_cleanup" }); } const stop = vi.fn(originalStop.bind(provider)); provider.stopLease = stop; if (outcome === "changed_binding") await db.update(environmentLeases).set({ providerLeaseId: "sandbox://fake/replacement" }) .where(eq(environmentLeases.id, acquired.lease.id)); await heartbeatService(db, { environmentRuntime: environmentRuntimeService(db) }).sweepPendingCleanupLeases({ backoffMs: 0 }); if (outcome === "changed_binding") { expect(stop).not.toHaveBeenCalled(); expect(await environmentService(db).getLeaseById(acquired.lease.id)).toMatchObject({ status: "pending_cleanup" }); } else { expect(stop).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ providerLeaseId: acquired.lease.providerLeaseId })); expect(await environmentService(db).getLeaseById(acquired.lease.id)).toMatchObject({ status: "released", cleanupStatus: "success" }); expect(await remoteExecutionHasStopped(db, companyId, runId)).toBe(true); } expect(release).not.toHaveBeenCalled(); expect(destroy).not.toHaveBeenCalled(); } finally { provider.stopLease = originalStop; } }, ); it("releases the remote sandbox when the lease insert rejects a foreign-company binding", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false, }, }); // A managed reconciliation binds the environment to another company after the // route guard read an empty binding list. The lease insert then rejects the // foreign-company binding with the 403 `environment_company_mismatch`. const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co", issuePrefix: "OTA", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); const destroySpy = vi.spyOn(sandboxProviderRuntime, "destroySandboxProviderLease"); try { await expect( runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }), ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" }, }); // The acquire records the durable pending-cleanup row before the teardown, // so the row lands while the database is proven reachable. The successful // teardown then releases the row to the terminal `expired` state, so no // active or pending_cleanup row remains for the orphan. const leaseRows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(leaseRows).toHaveLength(1); expect(leaseRows[0]?.status).toBe("expired"); expect(leaseRows[0]?.cleanupStatus).toBe("success"); // The acquire already provisioned the remote sandbox, so it releases the // sandbox on the rejection. Without this teardown the rejected insert leaks // a live sandbox that no lease row tracks. The teardown carries the // provisioned provider lease id. expect(destroySpy).toHaveBeenCalledTimes(1); expect(destroySpy.mock.calls[0]?.[0]?.providerLeaseId).toMatch( new RegExp(`^sandbox://fake/${runId}/[0-9a-f-]{36}$`), ); } finally { destroySpy.mockRestore(); } }); it("destroys the remote plugin sandbox when the lease insert rejects a foreign-company binding", async () => { // The Claude login runs on a plugin-backed sandbox provider (Daytona), so this // path is the one the login uses. It must also release the remote sandbox when // the conditional lease insert rejects a foreign-company binding. const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const pluginConfig = { provider: "fake-plugin", image: "fake:test", reuseLease: false }; const environment = { ...baseEnvironment, name: "Foreign-bound Plugin Sandbox", driver: "sandbox", config: pluginConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: pluginConfig, }); await insertSandboxProviderPlugin(pluginId); // A managed reconciliation binds the environment to another company after the // route guard read an empty binding list. The lease insert then rejects the // foreign-company binding with the 403 `environment_company_mismatch`. const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co", issuePrefix: "OTB", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), getWorker: vi.fn(() => ({ supportedMethods: ["environmentDestroyLease"] })), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "plugin-lease-1", metadata: { provider: "fake-plugin", image: "fake:test", reuseLease: false }, }; } if (method === "environmentDestroyLease") { return { providerLeaseId: "plugin-lease-1", state: "destroyed" }; } throw new Error(`Unexpected plugin method: ${method}`); }), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await expect( runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }), ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" }, }); // The acquire records the durable pending-cleanup row before the teardown, // so the row lands while the database is proven reachable. The successful // teardown then releases the row to the terminal `expired` state, so no // active or pending_cleanup row remains for the orphan. const leaseRows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(leaseRows).toHaveLength(1); expect(leaseRows[0]?.status).toBe("expired"); expect(leaseRows[0]?.cleanupStatus).toBe("success"); expect(leaseRows[0]?.metadata?.remoteExecutionTermination).toMatchObject({ companyId, runId, leaseId: leaseRows[0]!.id, providerLeaseId: "plugin-lease-1", state: "destroyed", }); // The acquire provisioned the remote plugin sandbox, so it destroys the // sandbox on the rejection. Without this teardown the rejected insert leaks a // live sandbox that no lease row tracks. const destroyCalls = (workerManager.call as unknown as ReturnType).mock.calls.filter( (callArgs) => callArgs[1] === "environmentDestroyLease", ); expect(destroyCalls).toHaveLength(1); expect(destroyCalls[0]?.[2]).toMatchObject({ providerLeaseId: "plugin-lease-1" }); await environmentService(db).update(environment.id, { driver: "local", config: {} }); }); it("records a durable pending-cleanup lease when the built-in teardown fails after a foreign-company rejection", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox Teardown Fail", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Built-in", issuePrefix: "OTC", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); // The remote teardown fails, so the acquire cannot release the sandbox // directly. It must record a durable pending-cleanup row instead. const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValue(new Error("teardown failed")); try { await expect( runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }), ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); expect(destroySpy).toHaveBeenCalledTimes(1); // The failed teardown left a durable pending-cleanup row that carries the // provider lease id for a later sweep. const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(1); expect(rows[0]?.status).toBe("pending_cleanup"); expect(rows[0]?.cleanupStatus).toBe("failed"); expect(rows[0]?.providerLeaseId).toMatch(new RegExp(`^sandbox://fake/${runId}/[0-9a-f-]{36}$`)); } finally { destroySpy.mockRestore(); } }); it("records the durable pending-cleanup row before it runs the inline teardown", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox Write First", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Write First", issuePrefix: "OWF", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); // The teardown reads the lease table when it runs. A durable pending_cleanup // row must already exist, which proves the acquire records the row before the // teardown. This ordering closes the window the finding describes: the durable // write lands while the database is proven reachable, before the teardown RPC // that a database outage could otherwise interrupt. let pendingCleanupRowsAtTeardown = -1; const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockImplementation(async () => { const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); pendingCleanupRowsAtTeardown = rows.filter((row) => row.status === "pending_cleanup").length; }); try { await expect( runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }), ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); // The durable row already tracked the orphan when the teardown ran. expect(pendingCleanupRowsAtTeardown).toBe(1); expect(destroySpy).toHaveBeenCalledTimes(1); // The teardown succeeded, so the acquire released the row to the terminal // `expired` state and left no pending_cleanup row. const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(1); expect(rows[0]?.status).toBe("expired"); expect(rows[0]?.cleanupStatus).toBe("success"); } finally { destroySpy.mockRestore(); } }); it("leaves no orphan and no error when the durable write fails but the teardown succeeds", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox Write Fail Teardown Ok", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Write Fail", issuePrefix: "OWK", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); // Force every durable pending-cleanup write to fail, but let the teardown // succeed. A successful teardown removes the orphan, so the acquire needs no // durable row and raises no orphan-cleanup write error. The write-first order // never turns a clean teardown into a failure. const realEnvironmentService = environmentsModule.environmentService; const factorySpy = vi .spyOn(environmentsModule, "environmentService") .mockImplementation((database: Parameters[0]) => { const real = realEnvironmentService(database); return { ...real, insertPendingCleanupLease: () => Promise.reject(new Error("pending-cleanup write failed; database down")), }; }); const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockResolvedValue(undefined as never); try { // Set the retry backoff to zero, so the failed write retries never slow the // test. const runtime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 }); const rejection = await runtime .acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }) .then( () => { throw new Error("acquireRunLease resolved but must reject"); }, (error: unknown) => error, ); // The rejection is the original company mismatch, not an orphan-cleanup // write error, because the teardown removed the orphan. expect(rejection).not.toBeInstanceOf(SandboxOrphanCleanupWriteError); expect(rejection).toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); expect(destroySpy).toHaveBeenCalledTimes(1); // No orphan remains and no durable row was needed, so the lease table is // empty for the environment. const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(0); // The buffer holds no orphan, so a flush recovers nothing. const flushed = await runtime.flushDeferredOrphanCleanups(); expect(flushed).toEqual({ recovered: 0, pending: 0 }); } finally { destroySpy.mockRestore(); factorySpy.mockRestore(); } }); it("records a durable pending-cleanup lease when the plugin teardown fails after a foreign-company rejection", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const pluginConfig = { provider: "fake-plugin", image: "fake:test", reuseLease: false }; const environment = { ...baseEnvironment, name: "Foreign-bound Plugin Sandbox Teardown Fail", driver: "sandbox", config: pluginConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: pluginConfig, }); await insertSandboxProviderPlugin(pluginId); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Plugin", issuePrefix: "OTD", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), getWorker: vi.fn(() => ({ supportedMethods: ["environmentDestroyLease"] })), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "plugin-lease-2", metadata: { provider: "fake-plugin", image: "fake:test", reuseLease: false }, }; } if (method === "environmentDestroyLease") { throw new Error("destroy failed"); } throw new Error(`Unexpected plugin method: ${method}`); }), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await expect( runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }), ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); // The failed plugin teardown left a durable pending-cleanup row that carries // the provider lease id for a later sweep. const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(1); expect(rows[0]?.status).toBe("pending_cleanup"); expect(rows[0]?.cleanupStatus).toBe("failed"); expect(rows[0]?.providerLeaseId).toBe("plugin-lease-2"); await environmentService(db).update(environment.id, { driver: "local", config: {} }); }); it("throws a SandboxOrphanCleanupWriteError when the built-in durable pending-cleanup write also fails", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox Cleanup Write Fail", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Cleanup Write", issuePrefix: "OCW", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); // The remote teardown fails, so the acquire tries to record a durable // pending-cleanup row instead. const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValue(new Error("teardown failed")); // Force the durable pending-cleanup write to fail on every attempt. The // write retries a few times, so reject the insert each time and let the // primary company-bound acquire reject for real. const failingInsert = vi .fn["insertPendingCleanupLease"]>, Promise>() .mockRejectedValue(new Error("pending-cleanup write failed")); const realEnvironmentService = environmentsModule.environmentService; const factorySpy = vi .spyOn(environmentsModule, "environmentService") .mockImplementation((database: Parameters[0]) => { const real = realEnvironmentService(database); return { ...real, insertPendingCleanupLease: failingInsert, }; }); // The durable database write fails, so the only durable handle left is the // error log. Capture it to prove the leaked sandbox stays discoverable. const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger); try { // Set the retry backoff to zero, so the retries never slow the test. const runtimeWithFailingCleanup = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0, }); const rejection = await runtimeWithFailingCleanup .acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }) .then( () => { throw new Error("acquireRunLease resolved but must reject"); }, (error: unknown) => error, ); // The failed durable write is not swallowed: the acquire throws a // SandboxOrphanCleanupWriteError that keeps the original 403 rejection as // its cause. expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError); expect(rejection).toMatchObject({ provider: "fake" }); expect((rejection as SandboxOrphanCleanupWriteError).cause).toMatchObject({ status: 403, details: { code: "environment_company_mismatch" }, }); expect(destroySpy).toHaveBeenCalledTimes(1); // The write retried the durable insert before it gave up, so more than one // attempt ran. expect(failingInsert.mock.calls.length).toBeGreaterThan(1); // The durable write failed before it created a row, so no lease row exists. const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(0); // The durable log preserves the provider identifiers, so an operator keeps // a handle to find and tear the leaked sandbox down by hand. const cleanupLog = logSpy.mock.calls.find( ([fields]) => (fields as { errorKind?: string } | undefined)?.errorKind === "sandbox_orphan_cleanup_write_failed", ); expect(cleanupLog).toBeDefined(); expect(cleanupLog?.[0]).toMatchObject({ provider: "fake", companyId, environmentId: environment.id, }); // The log never carries the caught exception, so no credential leaks. expect(cleanupLog?.[0]).not.toHaveProperty("cause"); expect(cleanupLog?.[0]).not.toHaveProperty("cleanupWriteError"); } finally { logSpy.mockRestore(); factorySpy.mockRestore(); destroySpy.mockRestore(); } }); it("recovers cleanup state when a retry of the durable pending-cleanup write succeeds", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox Cleanup Write Retry", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Cleanup Retry", issuePrefix: "OCR", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); // The remote teardown fails, so the acquire tries to record a durable // pending-cleanup row instead. const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValue(new Error("teardown failed")); // The first durable write attempt fails, so the acquire retries. The second // attempt delegates to the real insert, so a durable pending_cleanup row // lands and a later sweep can find the orphan. let insertAttempts = 0; const realEnvironmentService = environmentsModule.environmentService; const factorySpy = vi .spyOn(environmentsModule, "environmentService") .mockImplementation((database: Parameters[0]) => { const real = realEnvironmentService(database); return { ...real, insertPendingCleanupLease: ( input: Parameters[0], ) => { insertAttempts += 1; if (insertAttempts === 1) { return Promise.reject(new Error("pending-cleanup write failed once")); } return real.insertPendingCleanupLease(input); }, }; }); try { // Set the retry backoff to zero, so the retry never slows the test. const runtimeWithRetry = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0, }); const rejection = await runtimeWithRetry .acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }) .then( () => { throw new Error("acquireRunLease resolved but must reject"); }, (error: unknown) => error, ); // The retry landed the durable row, so the acquire rejects with the // original company-mismatch rejection, not a SandboxOrphanCleanupWriteError. expect(rejection).not.toBeInstanceOf(SandboxOrphanCleanupWriteError); expect(rejection).toMatchObject({ status: 403, details: { code: "environment_company_mismatch" }, }); expect(destroySpy).toHaveBeenCalledTimes(1); // The first attempt failed and the second succeeded, so the write ran twice. expect(insertAttempts).toBe(2); // A durable pending_cleanup lease row now tracks the orphan, so a sweep can // find and release the leaked sandbox. const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(1); expect(rows[0]?.status).toBe("pending_cleanup"); expect(rows[0]?.cleanupStatus).toBe("failed"); expect(rows[0]?.failureReason).toBe("acquire_rejected_teardown_failed"); } finally { factorySpy.mockRestore(); destroySpy.mockRestore(); } }); it("buffers the orphan in-process and a later sweep flush lands the durable row after the database recovers", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox Cleanup Buffer Flush", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Cleanup Buffer", issuePrefix: "OCB", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); // The remote teardown fails, so the acquire tries to record a durable // pending-cleanup row instead. const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValue(new Error("teardown failed")); // The database is down, so every synchronous pending-cleanup write rejects. // The flag flips to false when the database recovers, so a later flush lands // the durable row. let databaseDown = true; const realEnvironmentService = environmentsModule.environmentService; const factorySpy = vi .spyOn(environmentsModule, "environmentService") .mockImplementation((database: Parameters[0]) => { const real = realEnvironmentService(database); return { ...real, insertPendingCleanupLease: ( input: Parameters[0], ) => { if (databaseDown) { return Promise.reject(new Error("pending-cleanup write failed; database down")); } return real.insertPendingCleanupLease(input); }, }; }); const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger); try { // Set the retry backoff to zero, so the retries never slow the test. const runtime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 }); const rejection = await runtime .acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }) .then( () => { throw new Error("acquireRunLease resolved but must reject"); }, (error: unknown) => error, ); // The synchronous write failed on every attempt, so the acquire still // throws the orphan-cleanup write error that keeps the original rejection. expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError); // The database was down, so no durable row exists yet. let rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(0); // The acquire buffered the orphan in-process, so the log records the buffer. const bufferedLog = logSpy.mock.calls.find( ([fields]) => (fields as { errorKind?: string } | undefined)?.errorKind === "sandbox_orphan_cleanup_write_failed", ); expect(bufferedLog?.[0]).toMatchObject({ buffered: true }); // The database recovers, so the next cleanup-sweep flush lands the row. databaseDown = false; const flushed = await runtime.flushDeferredOrphanCleanups(); expect(flushed).toEqual({ recovered: 1, pending: 0 }); // A durable pending_cleanup row now tracks the orphan, so a sweep finds and // releases the leaked sandbox. rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(1); expect(rows[0]?.status).toBe("pending_cleanup"); expect(rows[0]?.cleanupStatus).toBe("failed"); expect(rows[0]?.failureReason).toBe("acquire_rejected_teardown_failed"); expect(rows[0]?.providerLeaseId).toBeTruthy(); // The buffer is empty now, so a second flush inserts nothing. const second = await runtime.flushDeferredOrphanCleanups(); expect(second).toEqual({ recovered: 0, pending: 0 }); const rowsAfter = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rowsAfter).toHaveLength(1); } finally { logSpy.mockRestore(); factorySpy.mockRestore(); destroySpy.mockRestore(); } }); it("persists the orphan to the durable spool so a restart recovers and lands the durable row", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox Cleanup Spool Restart", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Cleanup Spool Restart", issuePrefix: "OCS", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); // The remote teardown fails, so the acquire tries to record a durable // pending-cleanup row instead. const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValue(new Error("teardown failed")); // The database is down, so every synchronous pending-cleanup write rejects. // The flag flips to false to simulate the database recovering after a // restart. let databaseDown = true; const realEnvironmentService = environmentsModule.environmentService; const factorySpy = vi .spyOn(environmentsModule, "environmentService") .mockImplementation((database: Parameters[0]) => { const real = realEnvironmentService(database); return { ...real, insertPendingCleanupLease: ( input: Parameters[0], ) => { if (databaseDown) { return Promise.reject(new Error("pending-cleanup write failed; database down")); } return real.insertPendingCleanupLease(input); }, }; }); const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger); const spoolDir = process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR!; try { // The first process acquires, fails every synchronous write, and persists // the orphan to the durable spool. The retry backoff is zero, so the test // never waits. const firstProcessRuntime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 }); const rejection = await firstProcessRuntime .acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }) .then( () => { throw new Error("acquireRunLease resolved but must reject"); }, (error: unknown) => error, ); expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError); // The database was down, so no durable row exists yet. let rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(0); // The acquire persisted the orphan to the durable spool, so a file waits on // the disk and the log records the durable persist. const spoolFiles = (await readdir(spoolDir)).filter((name) => name.endsWith(".json")); expect(spoolFiles).toHaveLength(1); const persistedLog = logSpy.mock.calls.find( ([fields]) => (fields as { errorKind?: string } | undefined)?.errorKind === "sandbox_orphan_cleanup_write_failed", ); expect(persistedLog?.[0]).toMatchObject({ persisted: true }); // The process restarts. A fresh runtime keeps no in-process buffer, so only // the durable spool carries the orphan. The database recovers, so the first // cleanup-sweep flush loads the spool and lands the durable row. databaseDown = false; const restartedRuntime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 }); const flushed = await restartedRuntime.flushDeferredOrphanCleanups(); expect(flushed).toEqual({ recovered: 1, pending: 0 }); rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(1); expect(rows[0]?.status).toBe("pending_cleanup"); expect(rows[0]?.failureReason).toBe("acquire_rejected_teardown_failed"); expect(rows[0]?.providerLeaseId).toBeTruthy(); // The flush removed the spooled copy after the row landed, so a second // restart never re-inserts a duplicate. const spoolFilesAfter = (await readdir(spoolDir)).filter((name) => name.endsWith(".json")); expect(spoolFilesAfter).toHaveLength(0); const secondRestartRuntime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 }); const secondFlush = await secondRestartRuntime.flushDeferredOrphanCleanups(); expect(secondFlush).toEqual({ recovered: 0, pending: 0 }); const rowsAfter = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rowsAfter).toHaveLength(1); } finally { logSpy.mockRestore(); factorySpy.mockRestore(); destroySpy.mockRestore(); } }); it("buffers the orphan in-process when the durable spool write also fails", async () => { const idleBefore = idleWorkSnapshot().active; let releaseFlush!: () => void; let enteredFlush!: () => void; const flushGate = new Promise(resolve => { releaseFlush = resolve; }); const flushEntered = new Promise(resolve => { enteredFlush = resolve; }); const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox Cleanup Spool Unwritable", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Cleanup Spool Unwritable", issuePrefix: "OCU", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValue(new Error("teardown failed")); let databaseDown = true; const realEnvironmentService = environmentsModule.environmentService; const factorySpy = vi .spyOn(environmentsModule, "environmentService") .mockImplementation((database: Parameters[0]) => { const real = realEnvironmentService(database); return { ...real, insertPendingCleanupLease: ( input: Parameters[0], ) => { if (databaseDown) { return Promise.reject(new Error("pending-cleanup write failed; database down")); } enteredFlush(); return flushGate.then(() => real.insertPendingCleanupLease(input)); }, }; }); const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger); try { // Point the spool at an unwritable path, so the durable spool write fails // like a full or read-only disk. The acquire must fall back to the // in-process buffer and never lose the orphan silently. const unwritableSpool = { append: () => Promise.resolve(false), load: () => Promise.resolve([]), remove: () => Promise.resolve(), }; const runtime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0, orphanCleanupSpool: unwritableSpool, }); await runtime .acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }) .then( () => { throw new Error("acquireRunLease resolved but must reject"); }, () => undefined, ); // The spool write failed, so the log records the in-process-only fallback. const fallbackLog = logSpy.mock.calls.find( ([fields]) => (fields as { errorKind?: string } | undefined)?.errorKind === "sandbox_orphan_cleanup_write_failed", ); expect(fallbackLog?.[0]).toMatchObject({ persisted: false, buffered: true }); expect(idleWorkSnapshot().active).toBe(idleBefore + 1); // The same runtime still keeps the orphan in-process, so a flush after the // database recovers lands the durable row. databaseDown = false; const flushing = runtime.flushDeferredOrphanCleanups(); await flushEntered; // The batch was spliced out of the queue, but the DB does not own it yet. expect(idleWorkSnapshot().active).toBe(idleBefore + 1); releaseFlush(); const flushed = await flushing; expect(idleWorkSnapshot().active).toBe(idleBefore); expect(flushed).toEqual({ recovered: 1, pending: 0 }); const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(1); expect(rows[0]?.status).toBe("pending_cleanup"); } finally { releaseFlush(); logSpy.mockRestore(); factorySpy.mockRestore(); destroySpy.mockRestore(); } }); it("keeps the buffered orphan when the flush write still fails, then recovers on a later flush", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Foreign-bound Fake Sandbox Cleanup Buffer Requeue", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Cleanup Requeue", issuePrefix: "OCQ", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValue(new Error("teardown failed")); let databaseDown = true; const realEnvironmentService = environmentsModule.environmentService; const factorySpy = vi .spyOn(environmentsModule, "environmentService") .mockImplementation((database: Parameters[0]) => { const real = realEnvironmentService(database); return { ...real, insertPendingCleanupLease: ( input: Parameters[0], ) => { if (databaseDown) { return Promise.reject(new Error("pending-cleanup write failed; database down")); } return real.insertPendingCleanupLease(input); }, }; }); const errorLogSpy = vi.spyOn(logger, "error").mockImplementation(() => logger); const warnLogSpy = vi.spyOn(logger, "warn").mockImplementation(() => logger); try { const runtime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 }); await runtime .acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }) .then( () => { throw new Error("acquireRunLease resolved but must reject"); }, () => undefined, ); // The database is still down, so the flush re-queues the orphan instead of // losing it. The buffer keeps the record for a later tick. const firstFlush = await runtime.flushDeferredOrphanCleanups(); expect(firstFlush).toEqual({ recovered: 0, pending: 1 }); const rowsWhileDown = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rowsWhileDown).toHaveLength(0); // The flush logs the re-queue and never carries the caught write exception. // The sync-retry path also warns with the same error kind, so match on the // `requeued` field that only the flush warn carries. const requeueLog = warnLogSpy.mock.calls.find( ([fields]) => (fields as { requeued?: boolean } | undefined)?.requeued === true, ); expect(requeueLog).toBeDefined(); expect(requeueLog?.[0]).toMatchObject({ errorKind: "sandbox_orphan_cleanup_write_failed", requeued: true, }); expect(requeueLog?.[0]).not.toHaveProperty("cause"); expect(requeueLog?.[0]).not.toHaveProperty("cleanupWriteError"); // The database recovers, so the next flush lands the durable row exactly // once from the still-buffered record. databaseDown = false; const secondFlush = await runtime.flushDeferredOrphanCleanups(); expect(secondFlush).toEqual({ recovered: 1, pending: 0 }); const rowsAfter = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rowsAfter).toHaveLength(1); expect(rowsAfter[0]?.status).toBe("pending_cleanup"); } finally { warnLogSpy.mockRestore(); errorLogSpy.mockRestore(); factorySpy.mockRestore(); destroySpy.mockRestore(); } }); it("throws a SandboxOrphanCleanupWriteError when the plugin durable pending-cleanup write also fails", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const pluginConfig = { provider: "fake-plugin", image: "fake:test", reuseLease: false }; const environment = { ...baseEnvironment, name: "Foreign-bound Plugin Sandbox Cleanup Write Fail", driver: "sandbox", config: pluginConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: pluginConfig, }); await insertSandboxProviderPlugin(pluginId); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Plugin Cleanup Write", issuePrefix: "OPW", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), getWorker: vi.fn(() => ({ supportedMethods: ["environmentDestroyLease"] })), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "plugin-lease-write-fail", metadata: { provider: "fake-plugin", image: "fake:test", reuseLease: false }, }; } if (method === "environmentDestroyLease") { throw new Error("destroy failed"); } throw new Error(`Unexpected plugin method: ${method}`); }), } as unknown as PluginWorkerManager; const realEnvironmentService = environmentsModule.environmentService; const factorySpy = vi .spyOn(environmentsModule, "environmentService") .mockImplementation((database: Parameters[0]) => { const real = realEnvironmentService(database); return { ...real, insertPendingCleanupLease: () => Promise.reject(new Error("pending-cleanup write failed")), }; }); try { const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager, pendingCleanupWriteBackoffMs: 0, }); const rejection = await runtimeWithPlugin .acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }) .then( () => { throw new Error("acquireRunLease resolved but must reject"); }, (error: unknown) => error, ); expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError); expect(rejection).toMatchObject({ provider: "fake-plugin", providerLeaseId: "plugin-lease-write-fail" }); expect((rejection as SandboxOrphanCleanupWriteError).cause).toMatchObject({ status: 403, details: { code: "environment_company_mismatch" }, }); // The durable write failed before it created a row, so no lease row exists. const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(0); } finally { factorySpy.mockRestore(); } await environmentService(db).update(environment.id, { driver: "local", config: {} }); }); it("records the orphan directly in the pending_cleanup state with one atomic insert", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Atomic Pending Cleanup Insert", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const lease = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, executionWorkspaceId: null, issueId: null, heartbeatRunId: runId, provider: "fake", providerLeaseId: "sandbox://fake/atomic-orphan", metadata: { provider: "fake", reuseLease: false }, failureReason: "acquire_rejected_teardown_failed", }); // The returned lease is already in the terminal recovery state. It never // passes through the `active` state, so a crash cannot strand the orphan. expect(lease.status).toBe("pending_cleanup"); expect(lease.cleanupStatus).toBe("failed"); expect(lease.leasePolicy).toBe("ephemeral"); expect(lease.failureReason).toBe("acquire_rejected_teardown_failed"); expect(lease.providerLeaseId).toBe("sandbox://fake/atomic-orphan"); expect(lease.releasedAt).not.toBeNull(); // One insert created exactly one row, and the sweep reads that row directly. const rows = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)); expect(rows).toHaveLength(1); expect(rows[0]?.status).toBe("pending_cleanup"); expect(rows[0]?.cleanupStatus).toBe("failed"); }); it("tears the recorded built-in provider lease down for a pending-cleanup orphan", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Cleanup Retry Built-in", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Retry Built-in", issuePrefix: "OTE", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); // The first teardown fails, so the acquire records a pending-cleanup orphan. // The sweep then retries the teardown, which succeeds this time. const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValueOnce(new Error("teardown failed")) .mockResolvedValue(undefined); try { await expect( runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }), ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); const orphan = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)) .then((r) => r[0]!); expect(orphan.status).toBe("pending_cleanup"); const providerLeaseId = orphan.providerLeaseId; // The teardown accepts a null environment and reads the recorded provider // lease, so a delete or a provider change never strands it. const lease = await environmentService(db).getLeaseById(orphan.id); await runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! }); // The retry called the provider teardown a second time with the orphan // provider lease id. expect(destroySpy).toHaveBeenCalledTimes(2); expect(destroySpy).toHaveBeenLastCalledWith(expect.objectContaining({ providerLeaseId })); } finally { destroySpy.mockRestore(); } }); it("throws and keeps the pending-cleanup orphan when the teardown retry fails", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Cleanup Retry Scope", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Retry Scope", issuePrefix: "OTF", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValue(new Error("teardown failed")); try { await expect( runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }), ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); const orphan = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)) .then((r) => r[0]!); expect(orphan.status).toBe("pending_cleanup"); // The teardown retries, but the provider destroy still fails, so the // teardown throws. The caller keeps the orphan pending for a later sweep. const lease = await environmentService(db).getLeaseById(orphan.id); await expect( runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! }), ).rejects.toThrow(); const still = await db .select() .from(environmentLeases) .where(eq(environmentLeases.id, orphan.id)) .then((r) => r[0]!); expect(still.status).toBe("pending_cleanup"); expect(still.cleanupStatus).toBe("failed"); } finally { destroySpy.mockRestore(); } }); it("tears the recorded provider down after the environment provider changes", async () => { // Ordering: an acquire provisions a sandbox with provider A, the lease // insert rejects a foreign-company binding, and the compensating teardown // fails, so the acquire records an orphan for provider A. A provider change // then re-points the environment before the sweep runs. The sweep must tear // the orphan down from the recorded provider metadata, not the current // environment provider, so the change cannot strand the teardown. const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Provider Change Vs Orphan", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Provider Change", issuePrefix: "OTP", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockRejectedValueOnce(new Error("teardown failed")) .mockResolvedValue(undefined); try { await expect( runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }), ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); const orphan = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)) .then((r) => r[0]!); expect(orphan.status).toBe("pending_cleanup"); expect(orphan.provider).toBe("fake"); const providerLeaseId = orphan.providerLeaseId; // The provider-target mutation re-points the environment to a different // driver. The orphan teardown must not depend on this current config. await environmentService(db).update(environment.id, { driver: "local", config: {} }); // The teardown tears the recorded provider lease down, so it never reads // the re-pointed environment provider. const lease = await environmentService(db).getLeaseById(orphan.id); await runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! }); expect(destroySpy).toHaveBeenLastCalledWith(expect.objectContaining({ providerLeaseId })); } finally { destroySpy.mockRestore(); } }); it("reports the plugin cleanup worker not ready while its worker is down, then ready after it recovers", async () => { // A pending_cleanup orphan targets a plugin-backed provider. The sweep must // not consume a finite retry attempt while the plugin worker is briefly down. // So the readiness probe reports "not ready" for a down worker and "ready" // after the worker recovers. A built-in provider has no worker, so it is // always ready. const pluginId = randomUUID(); const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Cleanup Worker Readiness", config: { provider: "fake-plugin-ready", image: "fake:test", reuseLease: false }, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "paperclip.fake-plugin-ready-sandbox-provider", packageName: "@paperclipai/plugin-fake-ready-sandbox", displayName: "Fake Plugin Ready Sandbox Provider", description: "Test fake plugin provider readiness", environmentDrivers: [ { driverKey: "fake-plugin-ready", kind: "sandbox_provider", displayName: "Fake Plugin Ready", configSchema: { type: "object" }, }, ], }); // The worker is down at first, then recovers on the second probe. let workerRunning = false; const workerManager = { isRunning: vi.fn((id: string) => id === pluginId && workerRunning), call: vi.fn(async () => { throw new Error("call must not run during a readiness probe"); }), } as unknown as PluginWorkerManager; const orphan = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, executionWorkspaceId: null, issueId: null, heartbeatRunId: runId, provider: "fake-plugin-ready", providerLeaseId: "plugin-lease-readiness", metadata: { provider: "fake-plugin-ready", driver: "sandbox", reuseLease: false }, failureReason: "acquire_rejected_teardown_failed", }); const lease = (await environmentService(db).getLeaseById(orphan.id))!; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); // A down plugin worker is the transient restart window, so the probe reports // not ready. The sweep skips the lease without a claim, so no attempt burns. await expect( runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }), ).resolves.toBe(false); // The probe never calls the worker; it only checks the live worker state. expect((workerManager.call as ReturnType)).not.toHaveBeenCalled(); // The worker recovers, so the probe now reports ready and the sweep proceeds. workerRunning = true; await expect( runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }), ).resolves.toBe(true); // A built-in provider has no plugin worker, so its orphan is always ready. const builtinOrphan = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, executionWorkspaceId: null, issueId: null, heartbeatRunId: runId, provider: "fake", providerLeaseId: "builtin-lease-readiness", metadata: { provider: "fake", driver: "sandbox", reuseLease: false }, failureReason: "acquire_rejected_teardown_failed", }); const builtinLease = (await environmentService(db).getLeaseById(builtinOrphan.id))!; await expect( runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease: builtinLease }), ).resolves.toBe(true); }); it("reports the plugin cleanup provider not ready while the plugin is not ready, then ready after it recovers", async () => { // A pending_cleanup orphan targets a plugin-backed provider. A plugin reload // or a plugin reinstall moves the plugin through a "not ready" status. The // sweep must not consume a finite retry attempt in that transient window. So // the readiness probe reports "not ready" while the plugin status is not // "ready", even when its worker runs, and "ready" after the plugin recovers. const pluginId = randomUUID(); const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Cleanup Plugin Readiness", config: { provider: "fake-plugin-reload", image: "fake:test", reuseLease: false }, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "paperclip.fake-plugin-reload-sandbox-provider", packageName: "@paperclipai/plugin-fake-reload-sandbox", displayName: "Fake Plugin Reload Sandbox Provider", description: "Test fake plugin provider reload readiness", environmentDrivers: [ { driverKey: "fake-plugin-reload", kind: "sandbox_provider", displayName: "Fake Plugin Reload", configSchema: { type: "object" }, }, ], status: "installing", }); // The worker runs the whole time, so the probe result depends on the plugin // status alone, not on a down worker. const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async () => { throw new Error("call must not run during a readiness probe"); }), } as unknown as PluginWorkerManager; const orphan = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, executionWorkspaceId: null, issueId: null, heartbeatRunId: runId, provider: "fake-plugin-reload", providerLeaseId: "plugin-lease-reload", metadata: { provider: "fake-plugin-reload", driver: "sandbox", reuseLease: false }, failureReason: "acquire_rejected_teardown_failed", }); const lease = (await environmentService(db).getLeaseById(orphan.id))!; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); // A not-ready plugin is the transient reload window, so the probe reports // not ready. The sweep skips the lease without a claim, so no attempt burns. await expect( runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }), ).resolves.toBe(false); expect((workerManager.call as ReturnType)).not.toHaveBeenCalled(); // The plugin becomes ready, so the probe now reports ready and the sweep // proceeds. await db.update(plugins).set({ status: "ready" }).where(eq(plugins.id, pluginId)); await expect( runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }), ).resolves.toBe(true); }); it("reports the plugin cleanup provider not ready while the plugin is missing, then ready after it returns", async () => { // A pending_cleanup orphan targets a plugin-backed provider whose plugin row // is gone. A plugin reinstall removes and re-adds the plugin row, so the row // is missing for a short window. The sweep must not consume a finite retry // attempt in that window. So the readiness probe reports "not ready" while no // plugin row resolves the provider, and "ready" after the plugin returns. const pluginId = randomUUID(); const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Cleanup Plugin Missing", config: { provider: "fake-plugin-missing", image: "fake:test", reuseLease: false }, }); // The worker manager reports the plugin worker as running, so the probe // result depends on the missing plugin row alone, not on a down worker. const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async () => { throw new Error("call must not run during a readiness probe"); }), } as unknown as PluginWorkerManager; const orphan = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, executionWorkspaceId: null, issueId: null, heartbeatRunId: runId, provider: "fake-plugin-missing", providerLeaseId: "plugin-lease-missing", metadata: { provider: "fake-plugin-missing", driver: "sandbox", reuseLease: false }, failureReason: "acquire_rejected_teardown_failed", }); const lease = (await environmentService(db).getLeaseById(orphan.id))!; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); // No plugin row resolves the provider, so the probe reports not ready. The // sweep skips the lease without a claim, so no attempt burns while the plugin // reinstall is in flight. await expect( runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }), ).resolves.toBe(false); expect((workerManager.call as ReturnType)).not.toHaveBeenCalled(); // The plugin reinstall completes, so a ready plugin row now resolves the // provider and the probe reports ready. await insertSandboxProviderPlugin(pluginId, { pluginKey: "paperclip.fake-plugin-missing-sandbox-provider", packageName: "@paperclipai/plugin-fake-missing-sandbox", displayName: "Fake Plugin Missing Sandbox Provider", description: "Test fake plugin provider missing readiness", environmentDrivers: [ { driverKey: "fake-plugin-missing", kind: "sandbox_provider", displayName: "Fake Plugin Missing", configSchema: { type: "object" }, }, ], }); await expect( runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }), ).resolves.toBe(true); }); it("records an orphan with a null reference when a delete already removed the environment", async () => { // Ordering: a delete removes the environment before the acquire records the // orphan. The environment foreign key must not fail the insert. The record // persists with a null reference and its immutable provider metadata, so a // later sweep still tears the orphan down. const { companyId, environment } = await seedEnvironment({ driver: "sandbox", name: "Delete Before Orphan Record", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); // Remove the environment row first, so the record happens after the delete. await environmentService(db).remove(environment.id); const orphan = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, provider: "fake", providerLeaseId: "sandbox://fake/delete-before-record", metadata: { provider: "fake", image: "ubuntu:24.04", driver: "sandbox" }, failureReason: "acquire_rejected_teardown_failed", }); // The insert kept no environment reference, so the delete did not fail it. expect(orphan.environmentId).toBeNull(); expect(orphan.status).toBe("pending_cleanup"); const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockResolvedValue(undefined); try { // The recorded metadata drives the teardown, so a null environment tears // the orphan down. const lease = await environmentService(db).getLeaseById(orphan.id); await runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! }); expect(destroySpy).toHaveBeenCalledWith( expect.objectContaining({ providerLeaseId: "sandbox://fake/delete-before-record" }), ); } finally { destroySpy.mockRestore(); } }); it("keeps the orphan and tears it down after the environment is deleted", async () => { // Ordering: an orphan is recorded, then the environment is deleted. The // foreign key sets the reference null instead of cascading the row away, so // the orphan survives. The sweep tears it down from the recorded metadata. const { companyId, environment } = await seedEnvironment({ driver: "sandbox", name: "Delete After Orphan Record", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false }, }); const orphan = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, provider: "fake", providerLeaseId: "sandbox://fake/delete-after-record", metadata: { provider: "fake", image: "ubuntu:24.04", driver: "sandbox" }, failureReason: "acquire_rejected_teardown_failed", }); expect(orphan.environmentId).toBe(environment.id); // Delete the environment row directly. The foreign key sets the lease // reference null, so the orphan survives instead of being cascaded away. await environmentService(db).remove(environment.id); const survived = await db .select() .from(environmentLeases) .where(eq(environmentLeases.id, orphan.id)) .then((r) => r[0]!); expect(survived.environmentId).toBeNull(); expect(survived.status).toBe("pending_cleanup"); const destroySpy = vi .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease") .mockResolvedValue(undefined); try { // The recorded metadata drives the teardown, so a null environment tears // the orphan down after the delete set the reference null. const lease = await environmentService(db).getLeaseById(orphan.id); await runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! }); expect(destroySpy).toHaveBeenCalledWith( expect.objectContaining({ providerLeaseId: "sandbox://fake/delete-after-record" }), ); } finally { destroySpy.mockRestore(); } }); // A plugin provider whose config declares an `apiKey` secret-ref field. The // orphan teardown must resolve that recorded ref even when the environment // binding is gone, so these tests register a provider with a real secret-ref // field, unlike the plain `fake-plugin` tests above. const SECRET_REF_PLUGIN_KEY = "paperclip.secret-plugin-sandbox-provider"; const SECRET_REF_PROVIDER = "secret-plugin"; async function registerSecretRefPluginProvider(): Promise { const pluginId = randomUUID(); await db.insert(plugins).values({ id: pluginId, pluginKey: SECRET_REF_PLUGIN_KEY, packageName: "@paperclipai/plugin-secret-sandbox", version: "1.0.0", apiVersion: 1, categories: ["automation"], manifestJson: { id: SECRET_REF_PLUGIN_KEY, apiVersion: 1, version: "1.0.0", displayName: "Secret Plugin Sandbox Provider", description: "Test plugin provider with a secret-ref config field", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: SECRET_REF_PROVIDER, kind: "sandbox_provider", displayName: "Secret Plugin", configSchema: { type: "object", properties: { apiKey: { type: "string", format: "secret-ref" }, }, }, }, ], }, status: "ready", installOrder: 1, updatedAt: new Date(), } as any); return pluginId; } // A worker manager mock that records the config of each destroy call, so a // test can assert the resolved API key reached the provider teardown. function createDestroyRecordingWorkerManager(pluginId: string): { workerManager: PluginWorkerManager; destroyConfigs: Array | undefined>; } { const destroyConfigs: Array | undefined> = []; const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string, args: any) => { if (method === "environmentDestroyLease") { destroyConfigs.push(args?.config as Record | undefined); return {}; } throw new Error(`Unexpected plugin method: ${method}`); }), } as unknown as PluginWorkerManager; return { workerManager, destroyConfigs }; } async function createApiKeySecret(companyId: string, value: string): Promise { const secret = await secretService(db).create(companyId, { name: `sandbox-cleanup-api-key-${randomUUID()}`, provider: "local_encrypted", value, }); return secret.id; } it("tears down a secret-backed plugin orphan when a delete already removed the environment", async () => { // Ordering: a delete removed the environment (and its secret binding) before // the acquire recorded the orphan. The recorded config keeps the API-key // secret ref, so the teardown must resolve it without the environment // binding and send the resolved value to the provider destroy call. const pluginId = await registerSecretRefPluginProvider(); const { companyId, environment } = await seedEnvironment({ driver: "sandbox", name: "Secret Plugin Delete Before Record", config: { provider: SECRET_REF_PROVIDER, reuseLease: false }, }); const secretId = await createApiKeySecret(companyId, "old-provider-api-key"); await secretService(db).createBinding({ companyId, secretId, targetType: "environment", targetId: environment.id, configPath: "apiKey", }); // Remove the environment first, so the record happens after the delete. await environmentService(db).remove(environment.id); const orphan = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, provider: SECRET_REF_PROVIDER, providerLeaseId: "secret-plugin-lease-delete-before", metadata: { provider: SECRET_REF_PROVIDER, driver: "sandbox", sandboxProviderPlugin: true, pluginId, pluginKey: SECRET_REF_PLUGIN_KEY, apiKey: secretId, reuseLease: false, }, failureReason: "acquire_rejected_teardown_failed", }); expect(orphan.environmentId).toBeNull(); expect(orphan.status).toBe("pending_cleanup"); const { workerManager, destroyConfigs } = createDestroyRecordingWorkerManager(pluginId); const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const lease = await environmentService(db).getLeaseById(orphan.id); vi.mocked(workerManager.call).mockImplementationOnce(async (_pluginId, _method, args: any) => { destroyConfigs.push(args.config); return { providerLeaseId: args.providerLeaseId, state: "destroyed" }; }); await expect(runtimeWithPlugin.retryPendingSandboxTeardown({ environment: null, lease: lease! })) .resolves.toEqual({ providerLeaseId: orphan.providerLeaseId, state: "destroyed" }); expect(destroyConfigs).toHaveLength(1); // The recorded secret ref resolved to the old credential, and the resolved // value reached the provider teardown instead of the secret ref. expect(destroyConfigs[0]).toMatchObject({ apiKey: "old-provider-api-key" }); expect(destroyConfigs[0]?.apiKey).not.toBe(secretId); }); it("tears down a secret-backed plugin orphan after the environment is deleted", async () => { // Ordering: the orphan is recorded, then a delete removes the environment. // The foreign key sets the lease reference null, so the orphan survives. The // teardown resolves the recorded API-key ref without the environment. const pluginId = await registerSecretRefPluginProvider(); const { companyId, environment } = await seedEnvironment({ driver: "sandbox", name: "Secret Plugin Delete After Record", config: { provider: SECRET_REF_PROVIDER, reuseLease: false }, }); const secretId = await createApiKeySecret(companyId, "old-provider-api-key"); await secretService(db).createBinding({ companyId, secretId, targetType: "environment", targetId: environment.id, configPath: "apiKey", }); const orphan = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, provider: SECRET_REF_PROVIDER, providerLeaseId: "secret-plugin-lease-delete-after", metadata: { provider: SECRET_REF_PROVIDER, driver: "sandbox", sandboxProviderPlugin: true, pluginId, pluginKey: SECRET_REF_PLUGIN_KEY, apiKey: secretId, reuseLease: false, }, failureReason: "acquire_rejected_teardown_failed", }); expect(orphan.environmentId).toBe(environment.id); // Delete the environment. The foreign key sets the lease reference null, so // the orphan survives and the current binding is gone. await environmentService(db).remove(environment.id); const survived = await db .select() .from(environmentLeases) .where(eq(environmentLeases.id, orphan.id)) .then((r) => r[0]!); expect(survived.environmentId).toBeNull(); const { workerManager, destroyConfigs } = createDestroyRecordingWorkerManager(pluginId); const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const lease = await environmentService(db).getLeaseById(orphan.id); await runtimeWithPlugin.retryPendingSandboxTeardown({ environment: null, lease: lease! }); expect(destroyConfigs).toHaveLength(1); expect(destroyConfigs[0]).toMatchObject({ apiKey: "old-provider-api-key" }); }); it("tears down a secret-backed plugin orphan with the recorded credential after the provider is reconfigured", async () => { // Ordering: the orphan is recorded for the old credential, then the // environment is reconfigured to a new credential. The environment binding // now points to a different secret at the same path. The teardown must // resolve the OLD recorded ref, not the new binding, so it destroys the // sandbox the old credential provisioned. const pluginId = await registerSecretRefPluginProvider(); const { companyId, environment } = await seedEnvironment({ driver: "sandbox", name: "Secret Plugin Reconfigured", config: { provider: SECRET_REF_PROVIDER, reuseLease: false }, }); const oldSecretId = await createApiKeySecret(companyId, "old-provider-api-key"); const orphan = await environmentService(db).insertPendingCleanupLease({ companyId, environmentId: environment.id, provider: SECRET_REF_PROVIDER, providerLeaseId: "secret-plugin-lease-reconfigured", metadata: { provider: SECRET_REF_PROVIDER, driver: "sandbox", sandboxProviderPlugin: true, pluginId, pluginKey: SECRET_REF_PLUGIN_KEY, apiKey: oldSecretId, reuseLease: false, }, failureReason: "acquire_rejected_teardown_failed", }); expect(orphan.environmentId).toBe(environment.id); // Reconfigure the environment: a new secret replaces the binding at the same // path. The old recorded ref is no longer bound to the environment. const newSecretId = await createApiKeySecret(companyId, "new-provider-api-key"); await secretService(db).createBinding({ companyId, secretId: newSecretId, targetType: "environment", targetId: environment.id, configPath: "apiKey", }); await environmentService(db).update(environment.id, { driver: "sandbox", config: { provider: SECRET_REF_PROVIDER, apiKey: newSecretId, reuseLease: false }, }); const { workerManager, destroyConfigs } = createDestroyRecordingWorkerManager(pluginId); const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const lease = await environmentService(db).getLeaseById(orphan.id); await runtimeWithPlugin.retryPendingSandboxTeardown({ environment: null, lease: lease! }); expect(destroyConfigs).toHaveLength(1); // The teardown resolved the OLD recorded credential, not the new binding. expect(destroyConfigs[0]).toMatchObject({ apiKey: "old-provider-api-key" }); expect(destroyConfigs[0]?.apiKey).not.toBe("new-provider-api-key"); await environmentService(db).update(environment.id, { driver: "local", config: {} }); }); it("tears the recorded plugin provider lease down for a pending-cleanup orphan", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const pluginConfig = { provider: "fake-plugin", image: "fake:test", reuseLease: false }; const environment = { ...baseEnvironment, name: "Cleanup Retry Plugin", driver: "sandbox", config: pluginConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: pluginConfig, }); await insertSandboxProviderPlugin(pluginId); const otherCompanyId = randomUUID(); await db.insert(companies).values({ id: otherCompanyId, name: "Other Co Retry Plugin", issuePrefix: "OTG", status: "active", createdAt: new Date(), updatedAt: new Date(), }); await insertForeignCompanyEnvironmentBinding(otherCompanyId, environment); // The first destroy fails, so the acquire records a pending-cleanup orphan. // The sweep retries the destroy, which succeeds this time. let destroyAttempts = 0; const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), getWorker: vi.fn(() => ({ supportedMethods: ["environmentDestroyLease"] })), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "plugin-lease-3", metadata: { provider: "fake-plugin", image: "fake:test", reuseLease: false }, }; } if (method === "environmentDestroyLease") { destroyAttempts += 1; if (destroyAttempts === 1) { throw new Error("destroy failed"); } return {}; } throw new Error(`Unexpected plugin method: ${method}`); }), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await expect( runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, assertCompanyBinding: true, }), ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } }); const orphan = await db .select() .from(environmentLeases) .where(eq(environmentLeases.environmentId, environment.id)) .then((r) => r[0]!); expect(orphan.status).toBe("pending_cleanup"); expect(orphan.providerLeaseId).toBe("plugin-lease-3"); const lease = await environmentService(db).getLeaseById(orphan.id); await runtimeWithPlugin.retryPendingSandboxTeardown({ environment: null, lease: lease! }); expect(destroyAttempts).toBe(2); const destroyCall = (workerManager.call as unknown as ReturnType).mock.calls .filter((callArgs) => callArgs[1] === "environmentDestroyLease") .at(-1); expect(destroyCall?.[2]).toMatchObject({ providerLeaseId: "plugin-lease-3" }); await environmentService(db).update(environment.id, { driver: "local", config: {} }); }); it("uses plugin-backed sandbox config for execute and release", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const fakePluginConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, }; const environment = { ...baseEnvironment, name: "Fake Plugin Sandbox", driver: "sandbox", config: fakePluginConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: fakePluginConfig, }); await insertSandboxProviderPlugin(pluginId); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string, params: any) => { expect(params.config).toEqual(expect.objectContaining({ image: "fake:test", timeoutMs: 1234, reuseLease: false, })); expect(params.config).not.toHaveProperty("provider"); if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-1", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, remoteCwd: "/workspace", }, }; } if (method === "environmentExecute") { return { exitCode: 0, signal: null, timedOut: false, stdout: "ok\n", stderr: "", }; } if (method === "environmentReleaseLease") { expect(params.config).toEqual({ image: "fake:test", timeoutMs: 1234, reuseLease: false, }); expect(params.config).not.toHaveProperty("driver"); expect(params.config).not.toHaveProperty("executionWorkspaceMode"); expect(params.config).not.toHaveProperty("pluginId"); expect(params.config).not.toHaveProperty("pluginKey"); expect(params.config).not.toHaveProperty("providerMetadata"); expect(params.config).not.toHaveProperty("provider"); expect(params.config).not.toHaveProperty("sandboxProviderPlugin"); return undefined; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); const executed = await runtimeWithPlugin.execute({ environment, lease: acquired.lease, command: "printf", args: ["ok"], cwd: "/workspace", env: {}, timeoutMs: 1000, commandTimeoutMs: 900, forceSession: true, }); await environmentService(db).update(environment.id, { driver: "local", config: {}, }); const released = await runtimeWithPlugin.releaseRunLeases(runId); expect(executed.stdout).toBe("ok\n"); expect(released).toHaveLength(1); expect(released[0]?.lease.status).toBe("released"); // The execute call carries the optional log sink as the fifth argument; it // is undefined when the caller passes no sink. expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentExecute", expect.objectContaining({ timeoutMs: 1000, commandTimeoutMs: 900, bypassSession: false, }), 31000, undefined); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentReleaseLease", expect.anything(), 31234); }); // Build the fake plugin fixture for the run-parent release tests below. A // plugin sandbox provider can open a persistent session on the first command // and delete it on lease release; the delete emits a provider // `session.close` span. The host mints that span's parent from the active // step context at the release RPC. So the release must run under the run // parent, or the span loses its traceparent and the backend drops it. async function seedFakePluginSandbox() { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const fakePluginConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, }; const environment = { ...baseEnvironment, name: "Fake Plugin Sandbox", driver: "sandbox", config: fakePluginConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: fakePluginConfig, }); await insertSandboxProviderPlugin(pluginId); return { companyId, environment, runId, pluginId }; } it("runs a plugin-backed lease release under the run-time exec parent, so the teardown span keeps a valid traceparent", async () => { const { companyId, environment, runId, pluginId } = await seedFakePluginSandbox(); // The active step context observed at the moment the host issues the release // RPC. The real worker manager mints the provider-span traceparent from // exactly this value, so a non-null context with a valid parent proves the // teardown span keeps a host-minted parent. let releaseStepContext: ReturnType = null; const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-1", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, remoteCwd: "/workspace", }, }; } if (method === "environmentExecute") { return { exitCode: 0, signal: null, timedOut: false, stdout: "ok\n", stderr: "" }; } if (method === "environmentReleaseLease") { releaseStepContext = getActiveStepContext(); return undefined; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); // Build a real host run-parent context and run the first command under it, // exactly as the run drives an exec. The driver records this context for the // lease and replays it around the later release RPC. const runParent: StartupSpanContext = trace.setSpanContext(ROOT_CONTEXT, { traceId: "0af7651916cd43dd8448eb211c80319c", spanId: "b7ad6b7169203331", traceFlags: 1, isRemote: false, }); await runWithRuntimeParent(runParent, () => runtimeWithPlugin.execute({ environment, lease: acquired.lease, command: "printf", args: ["ok"], cwd: "/workspace", env: {}, timeoutMs: 1000, }), ); await environmentService(db).update(environment.id, { driver: "local", config: {} }); const released = await runtimeWithPlugin.releaseRunLeases(runId); expect(released).toHaveLength(1); // The release RPC ran under the recorded run parent, so the host mints a // valid W3C traceparent from it. A dropped span would show a null context. expect(releaseStepContext).not.toBeNull(); expect(releaseStepContext?.parentContext).toBe(runParent); expect(traceparentFromContextToken(releaseStepContext?.parentContext)).toBe( "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01", ); }); it("releases a plugin-backed lease with no active step context when the run drove no exec", async () => { const { companyId, environment, runId, pluginId } = await seedFakePluginSandbox(); let releaseStepContext: ReturnType = null; let releaseCalled = false; const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-1", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, remoteCwd: "/workspace", }, }; } if (method === "environmentReleaseLease") { releaseCalled = true; releaseStepContext = getActiveStepContext(); return undefined; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); await environmentService(db).update(environment.id, { driver: "local", config: {} }); const released = await runtimeWithPlugin.releaseRunLeases(runId); expect(released).toHaveLength(1); // No command ran, so no session opened and no teardown span is emitted. The // release must not invent a parent from an unrelated ambient context; it // runs unwrapped, exactly as before the fix. expect(releaseCalled).toBe(true); expect(releaseStepContext).toBeNull(); }); it("forwards the bypassSession flag to the plugin execute RPC so a pre-run command skips the session", async () => { const { companyId, environment, runId, pluginId } = await seedFakePluginSandbox(); // Capture the params of each environmentExecute RPC, so the test can assert // the host forwards `bypassSession` to the provider unchanged. const executeParams: Array> = []; const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string, params: Record) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-1", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, remoteCwd: "/workspace", }, }; } if (method === "environmentExecute") { executeParams.push(params); return { exitCode: 0, signal: null, timedOut: false, stdout: "ok\n", stderr: "" }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); // A pre-run command (the provision command) sets bypassSession explicitly. await runtimeWithPlugin.execute({ environment, lease: acquired.lease, command: "bash", args: ["-lc", "true"], cwd: "/workspace", env: {}, timeoutMs: 1000, bypassSession: true, }); // An in-run command runs under the run-parent context, so the host does not // bypass the session and the provider opens/uses the session. const runParent: StartupSpanContext = trace.setSpanContext(ROOT_CONTEXT, { traceId: "0af7651916cd43dd8448eb211c80319c", spanId: "b7ad6b7169203331", traceFlags: 1, isRemote: false, }); await runWithRuntimeParent(runParent, () => runtimeWithPlugin.execute({ environment, lease: acquired.lease, command: "printf", args: ["ok"], cwd: "/workspace", env: {}, timeoutMs: 1000, }), ); expect(executeParams).toHaveLength(2); expect(executeParams[0]?.bypassSession).toBe(true); // An in-run command carries a run parent, so it does not bypass the session. expect(executeParams[1]?.bypassSession).toBe(false); }); it("bypasses the session for a context-less command so the setup span keeps a run parent", async () => { const { companyId, environment, runId, pluginId } = await seedFakePluginSandbox(); // Capture the params of each environmentExecute RPC, so the test can assert // the host derives `bypassSession` from the active run-parent context. const executeParams: Array> = []; const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string, params: Record) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-1", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, remoteCwd: "/workspace", }, }; } if (method === "environmentExecute") { executeParams.push(params); return { exitCode: 0, signal: null, timedOut: false, stdout: "ok\n", stderr: "" }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); // A command with no active run-parent context (a pre-run install/probe or // the agent process launch) runs before the run trace is active. The host // mints no plugin RPC traceparent for it, so a session opened here would drop // its setup span. The host bypasses the session for such a command. await runtimeWithPlugin.execute({ environment, lease: acquired.lease, command: "sh", args: ["-c", "command -v claude"], cwd: "/workspace", env: {}, timeoutMs: 1000, }); expect(executeParams).toHaveLength(1); expect(executeParams[0]?.bypassSession).toBe(true); }); it("builds the workspace-realization record with referenced sources for a plugin-backed sandbox realize", async () => { // A provider plugin realize handler returns only its realized cwd and provider metadata; it does // not build the workspace-realization record. The server must build that record from the run // request, so the referenced (mentioned) project sources reach the adapter through // `realization.additional`. Without the record the sandbox agent never receives the mentioned // projects. This test drives the plugin-backed sandbox realize path and asserts the referenced // source survives into the returned record. const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const fakePluginConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, }; const environment = { ...baseEnvironment, name: "Fake Plugin Sandbox Realize", driver: "sandbox", config: fakePluginConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: fakePluginConfig, }); await insertSandboxProviderPlugin(pluginId); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-realize-1", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, remoteCwd: "/workspace", }, }; } if (method === "environmentRealizeWorkspace") { // Mimic a real provider (for example Daytona): return only the realized cwd and provider // metadata, never a `workspaceRealization` record. return { cwd: "/workspace/project", metadata: { provider: "fake-plugin", remoteCwd: "/workspace/project", }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); const workspaceRealizationRequest = createReferencedWorkspaceRealizationRequest(companyId, environment, runId); const realized = await runtimeWithPlugin.realizeWorkspace({ environment, lease: acquired.lease, workspace: { localPath: "/tmp/anchor", mode: "ephemeral", metadata: { workspaceRealizationRequest }, }, }); // The provider realized cwd and provider metadata survive. expect(realized.cwd).toBe("/workspace/project"); expect(realized.metadata?.provider).toBe("fake-plugin"); // The server-built record carries the referenced source through `additional`, so the adapter can // stage the mentioned project into the sandbox. const realization = realized.metadata?.workspaceRealization as Record | undefined; expect(realization).toBeDefined(); expect(realization?.additional).toEqual([ expect.objectContaining({ path: "/tmp/referenced-project", projectId: "referenced-project-1", }), ]); }); it("builds the workspace-realization record with referenced sources for a built-in sandbox realize", async () => { // The sandbox driver `realizeWorkspace` has two exits: a plugin-backed provider and a // built-in provider. Both must build the same workspace-realization record, so the // referenced (mentioned) project sources reach the adapter through `realization.additional`. // The test above covers the plugin exit. This test covers the built-in exit (no provider // plugin), so a regression on either exit fails a test. const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Fake Sandbox Realize", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: true, }, }); const acquired = await runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); const workspaceRealizationRequest = createReferencedWorkspaceRealizationRequest(companyId, environment, runId); const realized = await runtime.realizeWorkspace({ environment, lease: acquired.lease, workspace: { localPath: "/tmp/anchor", mode: "ephemeral", metadata: { workspaceRealizationRequest }, }, }); // The built-in exit builds the record and carries the referenced source through `additional`, // so the adapter can stage the mentioned project into the sandbox. const realization = realized.metadata?.workspaceRealization as Record | undefined; expect(realization).toBeDefined(); expect(realization?.additional).toEqual([ expect.objectContaining({ path: "/tmp/referenced-project", projectId: "referenced-project-1", }), ]); }); it("uses resolved secret-ref config for plugin-backed sandbox execute and release", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const apiSecret = await secretService(db).create(companyId, { name: `secure-plugin-api-key-${randomUUID()}`, provider: "local_encrypted", value: "resolved-provider-key", }); const providerConfig = { provider: "secure-plugin", template: "base", apiKey: apiSecret.id, timeoutMs: 1234, reuseLease: false, }; const environment = { ...baseEnvironment, name: "Secure Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await secretService(db).createBinding({ companyId, secretId: apiSecret.id, targetType: "environment", targetId: environment.id, configPath: "apiKey", }); await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "acme.secure-sandbox-provider", packageName: "@acme/secure-sandbox-provider", displayName: "Secure Sandbox Provider", description: "Test schema-driven provider", environmentDrivers: [ { driverKey: "secure-plugin", kind: "sandbox_provider", displayName: "Secure Sandbox", configSchema: { type: "object", properties: { template: { type: "string" }, apiKey: { type: "string", format: "secret-ref" }, timeoutMs: { type: "number" }, reuseLease: { type: "boolean" }, }, }, }, ], }); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string, params: any) => { expect(params.config.apiKey).toBe("resolved-provider-key"); expect(params.config).not.toHaveProperty("provider"); if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-1", metadata: { provider: "secure-plugin", template: "base", apiKey: "resolved-provider-key", timeoutMs: 1234, reuseLease: false, sandboxId: "sandbox-1", remoteCwd: "/workspace", }, }; } if (method === "environmentExecute") { return { exitCode: 0, signal: null, timedOut: false, stdout: "ok\n", stderr: "", }; } if (method === "environmentReleaseLease") { return undefined; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); expect(acquired.lease.metadata).toMatchObject({ provider: "secure-plugin", template: "base", apiKey: apiSecret.id, timeoutMs: 1234, sandboxId: "sandbox-1", }); const executed = await runtimeWithPlugin.execute({ environment, lease: acquired.lease, command: "printf", args: ["ok"], cwd: "/workspace", env: {}, }); await environmentService(db).update(environment.id, { driver: "local", config: {}, }); const released = await runtimeWithPlugin.releaseRunLeases(runId); expect(executed.stdout).toBe("ok\n"); expect(released).toHaveLength(1); expect(released[0]?.lease.status).toBe("released"); // The execute call carries the optional log sink as the fifth argument; it // is undefined when the caller passes no sink. expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentExecute", expect.objectContaining({ config: expect.objectContaining({ apiKey: "resolved-provider-key", }), }), 31234, undefined); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentReleaseLease", expect.objectContaining({ config: expect.objectContaining({ apiKey: "resolved-provider-key", }), }), 31234); }); it("resolves persisted secret_ref binding objects in sandbox provider config at runtime", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment } = await seedEnvironment(); const apiSecret = await secretService(db).create(companyId, { name: `secure-plugin-api-key-${randomUUID()}`, provider: "local_encrypted", value: "resolved-provider-key", }); const providerConfig = { provider: "secure-plugin", template: "base", apiKey: { type: "secret_ref", secretId: apiSecret.id, version: "latest" }, timeoutMs: 1234, reuseLease: false, }; const environment = { ...baseEnvironment, name: "Secure Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await secretService(db).createBinding({ companyId, secretId: apiSecret.id, targetType: "environment", targetId: environment.id, configPath: "apiKey", }); await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "acme.secure-sandbox-provider", packageName: "@acme/secure-sandbox-provider", displayName: "Secure Sandbox Provider", description: "Test schema-driven provider", environmentDrivers: [ { driverKey: "secure-plugin", kind: "sandbox_provider", displayName: "Secure Sandbox", configSchema: { type: "object", properties: { template: { type: "string" }, apiKey: { type: "string", format: "secret-ref" }, timeoutMs: { type: "number" }, reuseLease: { type: "boolean" }, }, }, }, ], }); const resolved = await resolveEnvironmentDriverConfigForRuntime(db, companyId, environment); expect(resolved.driver).toBe("sandbox"); expect(resolved.config).toMatchObject({ provider: "secure-plugin", template: "base", apiKey: "resolved-provider-key", }); }); it("waits briefly for a ready sandbox provider plugin worker to come online", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, }; const environment = { ...baseEnvironment, name: "Eventually Running Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "acme.eventually-running-sandbox-provider", packageName: "@acme/eventually-running-sandbox-provider", displayName: "Eventually Running Sandbox Provider", description: "Test plugin worker startup grace period", author: "Acme", }); let runningChecks = 0; const workerManager = { isRunning: vi.fn((id: string) => { if (id !== pluginId) return false; runningChecks += 1; return runningChecks >= 3; }), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-1", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager, pluginWorkerReadyTimeoutMs: 25, pluginWorkerReadyPollMs: 1, }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); expect(acquired.lease.providerLeaseId).toBe("sandbox-1"); expect(workerManager.isRunning).toHaveBeenCalledTimes(3); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234); }); it("throws a worker-not-running error once the readiness deadline is exhausted", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, }; const environment = { ...baseEnvironment, name: "Never Running Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "acme.never-running-sandbox-provider", packageName: "@acme/never-running-sandbox-provider", displayName: "Never Running Sandbox Provider", description: "Test plugin worker that never comes online before the readiness deadline", author: "Acme", }); const workerManager = { isRunning: vi.fn(() => false), call: vi.fn(async (_pluginId: string, method: string) => { throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager, pluginWorkerReadyTimeoutMs: 25, pluginWorkerReadyPollMs: 1, }); await expect( runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }), ).rejects.toThrow(/worker is not running/); // Confirms the loop actually polled repeatedly across the deadline window // instead of giving up after a single check. expect(workerManager.isRunning.mock.calls.length).toBeGreaterThanOrEqual(2); }); it("keeps polling across a longer worker restart window and succeeds once the handle registers late", async () => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, }; const environment = { ...baseEnvironment, name: "Registered Late Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "acme.registered-late-sandbox-provider", packageName: "@acme/registered-late-sandbox-provider", displayName: "Registered Late Sandbox Provider", description: "Test plugin worker handle that stays unregistered/starting for most of the readiness window", author: "Acme", }); // Simulates a worker process restart: the handle is absent (or "starting") // for most of the readiness window and only reports running with several // checks left before the deadline. A larger attempt count than the // existing "waits briefly" sanity test, to prove the loop is bound by the // deadline rather than by a small fixed number of attempts. const readyAfterChecks = 8; let checks = 0; const workerManager = { isRunning: vi.fn((id: string) => { if (id !== pluginId) return false; checks += 1; return checks >= readyAfterChecks; }), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-registered-late", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: false, }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager, pluginWorkerReadyTimeoutMs: 200, pluginWorkerReadyPollMs: 1, }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); expect(acquired.lease.providerLeaseId).toBe("sandbox-registered-late"); expect(checks).toBe(readyAfterChecks); }); it.each([ { label: "explicit provider and bridge budgets", config: { timeoutMs: 1_234, bridgeRequestTimeoutMs: 40_000 }, declared: 300_000, expected: 70_000 }, { label: "declared acquisition default", config: {}, declared: 300_000, expected: 330_000 }, { label: "explicit shorter provider budget", config: { timeoutMs: 5_000 }, declared: 300_000, expected: 35_000 }, { label: "bridge budget does not shorten acquisition default", config: { bridgeRequestTimeoutMs: 40_000 }, declared: 300_000, expected: 330_000 }, { label: "bridge extends the acquisition default", config: { bridgeRequestTimeoutMs: 400_000 }, declared: 300_000, expected: 430_000 }, { label: "undeclared provider retains worker default", config: {}, declared: undefined, expected: undefined }, ])("uses $label for plugin-backed sandbox acquisition", async ({ config, declared, expected }) => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", ...config, reuseLease: false, }; const environment = { ...baseEnvironment, name: "Long Lease Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "acme.long-lease-sandbox-provider", packageName: "@acme/long-lease-sandbox-provider", displayName: "Long Lease Sandbox Provider", description: "Test plugin worker acquire timeout", environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", defaultAcquireTimeoutMs: declared, // A timeoutMs schema default can mean lease lifetime (for example // E2B). Only the dedicated declaration sets the host RPC budget. configSchema: { type: "object", properties: { timeoutMs: { type: "number", default: 3_600_000 } } }, }, ], }); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "sandbox-1", metadata: { provider: "fake-plugin", image: "fake:test", ...config, reuseLease: false, }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); expect(acquired.lease.providerLeaseId).toBe("sandbox-1"); expect(workerManager.call).toHaveBeenCalledWith( pluginId, "environmentAcquireLease", expect.objectContaining({ driverKey: "fake-plugin", config: { image: "fake:test", ...config, reuseLease: false, }, }), expected, ); }); it("preserves the exact lease when plugin-backed sandbox resume throws", async () => { const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, }; const environment = { ...baseEnvironment, name: "Reusable Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "acme.fake-sandbox-provider", packageName: "@acme/fake-sandbox-provider", displayName: "Fake Sandbox Provider", description: "Test schema-driven provider", environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", supportsReusableLeases: true, configSchema: { type: "object", properties: { image: { type: "string" }, timeoutMs: { type: "number" }, reuseLease: { type: "boolean" }, }, }, }, ], }); const executionWorkspaceId = randomUUID(); const projectId = randomUUID(); await insertReusableProject(projectId, companyId); await insertReusableExecutionWorkspace(executionWorkspaceId, companyId, projectId); const staleLease = await insertStaleReusablePluginLease(companyId, environment, executionWorkspaceId, runId, agentId, pluginId, providerConfig); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentResumeLease") { throw new Error("stale sandbox"); } if (method === "environmentDestroyLease") { return undefined; } if (method === "environmentAcquireLease") { return { providerLeaseId: "fresh-plugin-lease", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, remoteCwd: "/workspace", }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await expect(runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId, heartbeatRunId: runId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, })).rejects.toThrow("the lease was preserved and no replacement was created"); expect(workerManager.call).toHaveBeenNthCalledWith(1, pluginId, "environmentResumeLease", expect.objectContaining({ driverKey: "fake-plugin", providerLeaseId: "stale-plugin-lease", }), 31234); expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentDestroyLease", expect.anything(), expect.anything(), ); expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentAcquireLease", expect.anything(), expect.anything(), ); await expect(environmentService(db).getLeaseById(staleLease.id)).resolves.toMatchObject({ status: "active", }); }); it("resumes only the exact stopped sandbox lifecycle without acquiring or reseeding", async () => { const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); await environmentService(db).releaseLease(seeded.reusableLease.id, "released", { cleanupStatus: "success" }); const lease = (await environmentService(db).getLeaseById(seeded.reusableLease.id))!; const call = vi.fn(async (_id: string, method: string) => { if (method !== "environmentResumeLease") throw new Error("Only exact resume is permitted"); return { providerLeaseId: lease.providerLeaseId, metadata: { remoteCwd: "/workspace" } }; }); const workerManager = { isRunning: () => true, call, getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] }), } as unknown as PluginWorkerManager; const runtime = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await expect(runtime.resumeRunLease({ environment: seeded.environment, lease })).resolves.toMatchObject({ providerLeaseId: lease.providerLeaseId }); expect(call).toHaveBeenCalledOnce(); expect(call).toHaveBeenCalledWith(seeded.pluginId, "environmentResumeLease", expect.objectContaining({ companyId: seeded.companyId, environmentId: seeded.environment.id, providerLeaseId: lease.providerLeaseId, leaseMetadata: lease.metadata, }), expect.any(Number)); expect((await environmentService(db).getLeaseById(lease.id))?.status).toBe("released"); expect(await db.select().from(environmentLeases)).toHaveLength(1); }); it.each(["stopped", "unconfirmed", "foreign_marker", "competing_owner", "retained_owner", "no_stop_capability", "live_request", "late_receipt", "duplicate_key", "owner_stopped", "owner_driver_changed", "owner_missing", "legacy_intent", "foreign_plugin_pin", "v2_missing_pin"])("recovers a pending export resume with stop-only cleanup after restart: %s", async outcome => { const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); const lease = seeded.reusableLease; const requestId = randomUUID(); if (["duplicate_key", "owner_stopped", "owner_driver_changed", "owner_missing", "legacy_intent"].includes(outcome)) { const [owner] = await db.select().from(plugins).where(eq(plugins.id, seeded.pluginId)); await db.insert(plugins).values({ ...owner, id: randomUUID(), pluginKey: "other.same-driver", packageName: "@other/same-driver", installOrder: -1, manifestJson: { ...owner.manifestJson, id: "other.same-driver" } }); if (outcome === "owner_driver_changed") await db.update(plugins).set({ manifestJson: { ...owner.manifestJson, environmentDrivers: [] } }).where(eq(plugins.id, seeded.pluginId)); if (outcome === "owner_missing") await db.delete(plugins).where(eq(plugins.id, seeded.pluginId)); } await db.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed", metadata: { ...lease.metadata, nativeWorkspaceExportResume: { schema: outcome === "v2_missing_pin" ? "paperclip.workspace-export-resume.v2" : "paperclip.workspace-export-resume.v1", requestId, companyId: seeded.companyId, ...(["legacy_intent", "v2_missing_pin"].includes(outcome) ? {} : { pluginId: outcome === "foreign_plugin_pin" ? randomUUID() : seeded.pluginId }), runId: outcome === "foreign_marker" ? randomUUID() : lease.heartbeatRunId, leaseId: lease.id, provider: lease.provider, providerLeaseId: lease.providerLeaseId, resultId: randomUUID() }, pendingCleanupAttemptId: requestId, pendingCleanupInFlight: true, pendingCleanupLeaseExpiresAtMs: Date.now() + (outcome === "live_request" ? 60_000 : -1), } }).where(eq(environmentLeases.id, lease.id)); if (["competing_owner", "retained_owner"].includes(outcome)) await db.insert(environmentLeases).values({ companyId: seeded.companyId, environmentId: seeded.environment.id, status: outcome === "retained_owner" ? "retained" : "active", provider: lease.provider, providerLeaseId: lease.providerLeaseId }); const call = vi.fn(async (_id: string, method: string) => { if (method !== "environmentStopLease") throw new Error("Saved workspace must never be destroyed"); if (outcome === "late_receipt") await db.update(environmentLeases).set({ status: "active", heartbeatRunId: null, cleanupStatus: null, metadata: { newOwner: true } }).where(eq(environmentLeases.id, lease.id)); return outcome === "unconfirmed" ? undefined : { providerLeaseId: lease.providerLeaseId, state: "stopped" }; }); const workerManager = { isRunning: (id: string) => !(outcome === "owner_stopped" && id === seeded.pluginId), call, getWorker: () => ({ supportedMethods: outcome === "no_stop_capability" ? ["environmentReleaseLease", "environmentDestroyLease"] : ["environmentStopLease", "environmentDestroyLease"] }), } as unknown as PluginWorkerManager; const restarted = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await heartbeatService(db, { environmentRuntime: restarted }).sweepPendingCleanupLeases({ backoffMs: 0 }); expect(call.mock.calls.some((entry) => entry[1] === "environmentDestroyLease")).toBe(false); const persisted = (await environmentService(db).getLeaseById(lease.id))!; if (["stopped", "duplicate_key", "legacy_intent"].includes(outcome)) { expect(call).toHaveBeenCalledWith(seeded.pluginId, "environmentStopLease", expect.objectContaining({ providerLeaseId: lease.providerLeaseId, cancelActiveWork: true }), expect.any(Number)); expect(persisted).toMatchObject({ status: "released", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "stopped", runId: lease.heartbeatRunId } } }); expect(persisted.metadata?.nativeWorkspaceExportResume).toBeUndefined(); } else if (outcome === "late_receipt") { expect(persisted).toMatchObject({ status: "active", heartbeatRunId: null, cleanupStatus: null, metadata: { newOwner: true } }); } else { expect(persisted).toMatchObject({ status: "pending_cleanup", cleanupStatus: "failed" }); expect(persisted.metadata?.remoteExecutionTermination).toBeUndefined(); if (["foreign_marker", "foreign_plugin_pin", "v2_missing_pin", "competing_owner", "retained_owner", "no_stop_capability", "live_request", "owner_stopped", "owner_driver_changed", "owner_missing"].includes(outcome)) expect(call).not.toHaveBeenCalled(); if (["owner_stopped", "owner_driver_changed", "owner_missing"].includes(outcome)) { expect(persisted.metadata?.pendingCleanupAttemptId).toBe(requestId); } } }); it.each(["stopped", "stop_failed", "unconfirmed", "restart", "committed_release", "inflight"])("preserves terminal ephemeral export work through release and restart: %s", async outcome => { const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); const lease = seeded.reusableLease, requestId = randomUUID(); await db.update(heartbeatRuns).set({ status: outcome === "committed_release" ? "succeeded" : "failed" }).where(eq(heartbeatRuns.id, seeded.runId)); await db.update(environmentLeases).set({ status: outcome === "committed_release" ? "active" : "pending_cleanup", leasePolicy: "ephemeral", cleanupStatus: "failed", metadata: { ...lease.metadata, reuseLease: false, nativeWorkspaceExportResume: { schema: "paperclip.workspace-export-resume.v2", purpose: "terminal_export", requestId, companyId: seeded.companyId, pluginId: seeded.pluginId, runId: seeded.runId, resultId: randomUUID(), leaseId: lease.id, provider: lease.provider, providerLeaseId: lease.providerLeaseId }, pendingCleanupAttemptId: requestId, pendingCleanupInFlight: outcome === "inflight", pendingCleanupLeaseExpiresAtMs: outcome === "inflight" ? Date.now() + 60_000 : 0, } }).where(eq(environmentLeases.id, lease.id)); let calls = 0; const call = vi.fn(async (_id: string, method: string, params: any) => { expect(method).toBe("environmentStopLease"); expect(params).toMatchObject({ providerLeaseId: lease.providerLeaseId, resourceDisposition: "stop_and_retain", config: { reuseLease: false } }); if (++calls === 1 && outcome === "stop_failed") throw new Error("injected stop failure"); if (calls === 1 && outcome === "unconfirmed") return undefined; return { providerLeaseId: lease.providerLeaseId, state: "stopped" }; }); const runtime = () => environmentRuntimeService(db, { pluginWorkerManager: { isRunning: () => true, call, getWorker: () => ({ supportedMethods: ["environmentStopLease", "environmentDestroyLease"] }), } as unknown as PluginWorkerManager }); if (outcome === "restart") await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 }); else await runtime().releaseRunLeases(seeded.runId, "failed", undefined, "stop_and_retain"); if (outcome === "inflight") { expect(call).not.toHaveBeenCalled(); return; } if (["stop_failed", "unconfirmed"].includes(outcome)) { expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "pending_cleanup", metadata: { pendingCleanupInFlight: false } }); await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 }); } expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "released", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "stopped" } } }); expect(call.mock.calls.every(entry => entry[1] === "environmentStopLease")).toBe(true); }); it("does not allocate a native-runner replacement without a verified backup", async () => { const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); const workerManager = { isRunning: vi.fn((id: string) => id === seeded.pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentResumeLease") { return { providerLeaseId: null, metadata: { expired: true } }; } if (method === "environmentDestroyLease") return undefined; if (method === "environmentAcquireLease") { throw new Error("replacement must not be allocated"); } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await expect(runtimeWithPlugin.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment, issueId: null, agentId: seeded.agentId, adapterType: "paperclip_runner", heartbeatRunId: seeded.runId, persistedExecutionWorkspace: { id: seeded.executionWorkspaceId, mode: "shared_workspace", }, })).rejects.toThrow("runner_harness_backup_unavailable"); expect(workerManager.call).not.toHaveBeenCalledWith( seeded.pluginId, "environmentAcquireLease", expect.anything(), expect.anything(), ); expect(workerManager.call).not.toHaveBeenCalledWith( seeded.pluginId, "environmentDestroyLease", expect.anything(), expect.anything(), ); }); it("permits native-runner replacement only after verifying the stamped backup", async () => { const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); const backupBase = await mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-replacement-")); const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR; process.env.PAPERCLIP_RUNNER_STATE_DIR = backupBase; try { const normalizedSessionId = "native-replacement-session"; const runnerInstanceId = "native-replacement-runner"; const sessionScopeId = "native-replacement-session-scope-v2"; const sessionRoot = path.join( backupBase, createHash("sha256").update(sessionScopeId).digest("hex"), ); const current = path.join(sessionRoot, "failover-backups", "current"); await mkdir(path.join(current, "runner"), { recursive: true }); await mkdir(path.join(current, "codex-home", "sessions"), { recursive: true }); await writeFile(path.join(current, "runner", "runner-state.json"), "runner-state"); await writeFile(path.join(current, "codex-home", "sessions", "thread.jsonl"), "thread-state"); const execution = { provider: { kind: "codex", model: null, approvalPolicy: "never" }, binding: { companyId: seeded.companyId, runId: seeded.runId, issueId: "issue", agentId: seeded.agentId, executionWorkspaceId: seeded.executionWorkspaceId, }, workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null }, session: { normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, }, } as never; const manifest = buildNativeHarnessBackupManifest({ backupRoot: current, execution, runnerInstanceId, providerSessionIdentity: { providerSessionId: "thread-1", providerBackendSessionId: "session-1", providerSessionIdentity: null, }, sourceProviderLeaseId: seeded.reusableLease.providerLeaseId!, }); const manifestPath = path.join(current, "manifest.json"); await writeFile(manifestPath, JSON.stringify(manifest)); const stamp = createNativeHarnessBackupStamp({ manifestPath, sessionScopeId, authorizedProviderLeaseId: seeded.reusableLease.providerLeaseId!, normalizedSessionId, runnerInstanceId, completedAt: manifest.completedAt, }); await environmentService(db).updateLeaseMetadata(seeded.reusableLease.id, { ...(seeded.reusableLease.metadata ?? {}), nativeHarnessBackup: stamp, }); const workerManager = { isRunning: vi.fn((id: string) => id === seeded.pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentResumeLease") { return { providerLeaseId: null, metadata: { expired: true } }; } if (method === "environmentDestroyLease") return undefined; if (method === "environmentAcquireLease") { return { providerLeaseId: "replacement-plugin-lease", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, remoteCwd: "/workspace", }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment, issueId: null, agentId: seeded.agentId, adapterType: "paperclip_runner", heartbeatRunId: seeded.runId, persistedExecutionWorkspace: { id: seeded.executionWorkspaceId, mode: "shared_workspace", }, }); expect(acquired.lease.providerLeaseId).toBe("replacement-plugin-lease"); expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "replacement", reason: "not_found", }); } finally { if (previousStateDirectory === undefined) { delete process.env.PAPERCLIP_RUNNER_STATE_DIR; } else { process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory; } await rm(backupBase, { recursive: true, force: true }); } }); it("fails closed and does not resume when a worker restart drops the resume method after the capability snapshot", async () => { const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, }; const environment = { ...baseEnvironment, name: "Reusable Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "acme.fake-sandbox-provider", packageName: "@acme/fake-sandbox-provider", displayName: "Fake Sandbox Provider", description: "Test schema-driven provider", environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", supportsReusableLeases: true, configSchema: { type: "object", properties: { image: { type: "string" }, timeoutMs: { type: "number" }, reuseLease: { type: "boolean" }, }, }, }, ], }); const executionWorkspaceId = randomUUID(); const projectId = randomUUID(); await insertReusableProject(projectId, companyId); await insertReusableExecutionWorkspace(executionWorkspaceId, companyId, projectId); const staleLease = await insertStaleReusablePluginLease(companyId, environment, executionWorkspaceId, runId, agentId, pluginId, providerConfig); // The runtime reads the worker methods once to decide reuse, then does // asynchronous database work before the resume dispatch. A worker restart // in that window drops `environmentResumeLease`. The first read returns the // reuse verbs, so the runtime treats the lease as resumable. Every later // read returns the restarted worker's methods, which no longer include // `environmentResumeLease`. let getWorkerReads = 0; const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentResumeLease") { // A regression that reads the stale snapshot dispatches the resume // RPC and fails here. The live worker cannot serve the method. throw new Error("worker no longer advertises environmentResumeLease"); } if (method === "environmentDestroyLease") { return undefined; } if (method === "environmentAcquireLease") { return { providerLeaseId: "fresh-plugin-lease", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, remoteCwd: "/workspace", }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => { getWorkerReads += 1; return { supportedMethods: getWorkerReads === 1 ? ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] : ["environmentReleaseLease", "environmentDestroyLease"], }; }), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await expect(runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId, heartbeatRunId: runId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, })).rejects.toThrow("the lease was preserved and no replacement was created"); // The runtime re-checks the live worker before the resume dispatch. The // restarted worker no longer advertises `environmentResumeLease`, so the // runtime must not dispatch the resume RPC. expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentResumeLease", expect.anything(), expect.anything(), ); // It preserves the stale reusable lease and does not allocate a replacement. expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentDestroyLease", expect.anything(), expect.anything(), ); expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentAcquireLease", expect.anything(), expect.anything(), ); await expect(environmentService(db).getLeaseById(staleLease.id)).resolves.toMatchObject({ status: "active", }); }); // Seed a reusable plugin sandbox lease that a worker created under an earlier // capability set. The worker restarts and no longer advertises the lifecycle // methods. The lease lifecycle paths must verify the live worker before they // dispatch a lifecycle RPC, so the runtime fails closed instead of a doomed // dispatch. async function seedStaleLifecycleReusableLease( leasePolicy: "reuse_by_environment" | "retain_on_failure" = "reuse_by_environment", ) { const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, }; const environment = { ...baseEnvironment, name: "Reusable Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "acme.fake-sandbox-provider", packageName: "@acme/fake-sandbox-provider", displayName: "Fake Sandbox Provider", description: "Test schema-driven provider", environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", supportsReusableLeases: true, configSchema: { type: "object" }, }, ], }); const executionWorkspaceId = randomUUID(); const projectId = randomUUID(); await db.insert(projects).values({ id: projectId, companyId, name: `Workspace ${projectId.slice(0, 8)}`, status: "active", createdAt: new Date(), updatedAt: new Date(), }); await db.insert(executionWorkspaces).values({ id: executionWorkspaceId, companyId, projectId, mode: "shared_workspace", strategyType: "project_primary", name: "Reusable workspace", status: "active", providerType: "local_fs", createdAt: new Date(), updatedAt: new Date(), }); const lease = await environmentService(db).acquireLease({ companyId, environmentId: environment.id, executionWorkspaceId, heartbeatRunId: runId, leasePolicy, provider: "fake-plugin", providerLeaseId: "stale-lifecycle-lease", metadata: { agentId, driver: "sandbox", pluginId, pluginKey: "acme.fake-sandbox-provider", sandboxProviderPlugin: true, provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, }, }); // The worker is running, but its discovery list dropped the reusable-lease // lifecycle methods. `call` throws on any lifecycle RPC so a regression that // dispatches one fails the test through the `not.toHaveBeenCalledWith` check. const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentAcquireLease", "environmentExecute"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); return { pluginId, environment, runId, lease, workerManager, runtimeWithPlugin }; } it("routes release to pending_cleanup when the worker no longer advertises the release lifecycle method", async () => { const { pluginId, lease, workerManager, runtimeWithPlugin } = await seedStaleLifecycleReusableLease(); const released = await runtimeWithPlugin.releaseRunLeases(lease.heartbeatRunId!); expect(released).toHaveLength(1); expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentReleaseLease", expect.anything(), expect.anything(), ); // The failed release verification must enter the pending-cleanup retry flow. // The reaper sweeps only `pending_cleanup` leases, so a `released` status // here would strand the still-active provider resource. await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({ status: "pending_cleanup", cleanupStatus: "failed", failureReason: "release_cleanup_failed", }); }); it("retains a retain_on_failure lease on failed release instead of routing to pending_cleanup", async () => { const { lease, runtimeWithPlugin } = await seedStaleLifecycleReusableLease("retain_on_failure"); const released = await runtimeWithPlugin.releaseRunLeases(lease.heartbeatRunId!, "failed"); expect(released).toHaveLength(1); // A retain_on_failure lease keeps the provider resource for reuse. The // reaper destroys `pending_cleanup` leases, so the retained lease must not // enter that flow even when the release verification fails. await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({ status: "retained", cleanupStatus: "failed", }); }); it("destroys a failed retained lease when the agent stops", async () => { const { lease, workerManager, runtimeWithPlugin, runId } = await seedStaleLifecycleReusableLease("retain_on_failure"); await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); await runtimeWithPlugin.releaseRunLeases(runId, "failed"); await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({ status: "retained", cleanupStatus: "failed" }); vi.mocked(workerManager.getWorker).mockReturnValue({ supportedMethods: ["environmentDestroyLease"] } as never); vi.mocked(workerManager.call).mockResolvedValue({ providerLeaseId: lease.providerLeaseId, state: "destroyed" }); const { heartbeatService } = await import("../services/heartbeat.js"); const stopped = await heartbeatService(db, { pluginWorkerManager: workerManager }) .stopInvocationsForAgents([lease.metadata!.agentId as string], "Agent lifecycle stop requested"); expect(stopped).toBe(true); expect(workerManager.call).toHaveBeenCalledWith(expect.any(String), "environmentDestroyLease", expect.objectContaining({ providerLeaseId: lease.providerLeaseId }), expect.any(Number)); await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({ status: "expired", cleanupStatus: "success" }); }); it("fails closed on expiry destruction when the worker no longer advertises the destroy lifecycle method", async () => { const { pluginId, lease, workerManager, runtimeWithPlugin } = await seedStaleLifecycleReusableLease(); await runtimeWithPlugin.releaseRunLeases(lease.heartbeatRunId!, "expired"); expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentDestroyLease", expect.anything(), expect.anything(), ); await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({ status: "pending_cleanup", cleanupStatus: "failed", }); }); it("does not resume released reusable plugin sandbox leases after provider config drift", async () => { const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "template-a", timeoutMs: 1234, reuseLease: true, }; const environment = { ...baseEnvironment, name: "Reusable Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "acme.fake-sandbox-provider", packageName: "@acme/fake-sandbox-provider", displayName: "Fake Sandbox Provider", description: "Test schema-driven provider", environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", supportsReusableLeases: true, configSchema: { type: "object", properties: { image: { type: "string" }, timeoutMs: { type: "number" }, reuseLease: { type: "boolean" }, }, }, }, ], }); const executionWorkspaceId = randomUUID(); const projectId = randomUUID(); await insertReusableProject(projectId, companyId); await insertReusableExecutionWorkspace(executionWorkspaceId, companyId, projectId); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string, params: any) => { if (method === "environmentAcquireLease") { return { providerLeaseId: `lease-${params.config.image}`, metadata: { provider: "fake-plugin", image: params.config.image, timeoutMs: params.config.timeoutMs, reuseLease: true, remoteCwd: "/workspace", }, }; } if (method === "environmentReleaseLease" || method === "environmentDestroyLease") { return undefined; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const first = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId, heartbeatRunId: runId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, }); expect(first.lease.metadata?.reusableSandboxLease).toMatchObject({ provider: "fake-plugin", leaseFingerprint: expect.objectContaining({ category: "lease", fingerprint: expect.stringMatching(/^v1:sha256:[a-f0-9]{64}$/), }), }); await runtimeWithPlugin.releaseRunLeases(runId); const nextRunId = randomUUID(); await insertRunningManualRun(nextRunId, companyId, agentId); const updatedEnvironment = { ...environment, config: { ...providerConfig, image: "template-b", }, }; await environmentService(db).update(environment.id, { config: updatedEnvironment.config, }); const second = await runtimeWithPlugin.acquireRunLease({ companyId, environment: updatedEnvironment, issueId: null, agentId, heartbeatRunId: nextRunId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, }); expect(second.lease.providerLeaseId).toBe("lease-template-b"); expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentResumeLease", expect.anything(), expect.anything(), ); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease", expect.objectContaining({ providerLeaseId: "lease-template-a", }), 31234); await expect(environmentService(db).getLeaseById(first.lease.id)).resolves.toMatchObject({ status: "expired", cleanupStatus: "success", failureReason: "lease_fingerprint_mismatch", }); }); it("does not resume released reusable plugin sandbox leases after secret version drift", async () => { const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); const apiSecret = await secretService(db).create(companyId, { name: `secure-plugin-api-key-${randomUUID()}`, provider: "local_encrypted", value: "resolved-provider-key", }); const providerConfig = { provider: "secure-plugin", template: "base", apiKey: apiSecret.id, timeoutMs: 1234, reuseLease: true, }; const environment = { ...baseEnvironment, name: "Secure Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await secretService(db).createBinding({ companyId, secretId: apiSecret.id, targetType: "environment", targetId: environment.id, configPath: "apiKey", }); await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "acme.secure-sandbox-provider", packageName: "@acme/secure-sandbox-provider", displayName: "Secure Sandbox Provider", description: "Test schema-driven provider", environmentDrivers: [ { driverKey: "secure-plugin", kind: "sandbox_provider", displayName: "Secure Sandbox", supportsReusableLeases: true, configSchema: { type: "object", properties: { template: { type: "string" }, apiKey: { type: "string", format: "secret-ref" }, timeoutMs: { type: "number" }, reuseLease: { type: "boolean" }, }, }, }, ], }); const executionWorkspaceId = randomUUID(); const projectId = randomUUID(); await insertReusableProject(projectId, companyId); await insertReusableExecutionWorkspace(executionWorkspaceId, companyId, projectId); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string, params: any) => { if (method === "environmentAcquireLease") { return { providerLeaseId: `lease-${params.config.apiKey}`, metadata: { provider: "secure-plugin", template: params.config.template, apiKey: params.config.apiKey, timeoutMs: params.config.timeoutMs, reuseLease: true, remoteCwd: "/workspace", }, }; } if (method === "environmentReleaseLease" || method === "environmentDestroyLease") { return undefined; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const first = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId, heartbeatRunId: runId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, }); await runtimeWithPlugin.releaseRunLeases(runId); await secretService(db).rotate(apiSecret.id, { value: "rotated-provider-key" }); const nextRunId = randomUUID(); await insertRunningManualRun(nextRunId, companyId, agentId); const second = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId, heartbeatRunId: nextRunId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, }); expect(second.lease.providerLeaseId).toBe("lease-rotated-provider-key"); expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentResumeLease", expect.anything(), expect.anything(), ); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease", expect.objectContaining({ providerLeaseId: "lease-resolved-provider-key", }), 31234); await expect(environmentService(db).getLeaseById(first.lease.id)).resolves.toMatchObject({ status: "expired", cleanupStatus: "success", failureReason: "lease_fingerprint_mismatch", }); const firstMetadata = JSON.stringify(first.lease.metadata); expect(firstMetadata).not.toContain("resolved-provider-key"); expect(firstMetadata).not.toContain("rotated-provider-key"); }); it("preserves active reusable sandbox leases held by another running run", async () => { const { pluginId, companyId, agentId, environment, executionWorkspaceId, reusableLease } = await seedReusablePluginSandboxLease(); const nextRunId = randomUUID(); await insertRunningManualRun(nextRunId, companyId, agentId); const workerManager = createFreshPluginLeaseWorker(pluginId); const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId, heartbeatRunId: nextRunId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, }); expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease"); expect(workerManager.call).toHaveBeenCalledOnce(); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.objectContaining({ agentId, executionWorkspaceId, runId: nextRunId, }), 31234); await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ status: "active", cleanupStatus: null, }); }); it("does not retain or resume plugin-backed sandbox leases unless the provider opts in", async () => { const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, }; const environment = { ...baseEnvironment, name: "Non-reusable Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "acme.non-reusable-sandbox-provider", packageName: "@acme/non-reusable-sandbox-provider", displayName: "Non-reusable Sandbox Provider", description: "Test provider without reusable lease support", }); const executionWorkspaceId = randomUUID(); const projectId = randomUUID(); await insertReusableProject(projectId, companyId); await db.insert(executionWorkspaces).values({ id: executionWorkspaceId, companyId, projectId, mode: "shared_workspace", strategyType: "project_primary", name: "Non-reusable workspace", status: "active", providerType: "local_fs", createdAt: new Date(), updatedAt: new Date(), }); await insertOldReusablePluginLease(companyId, environment, executionWorkspaceId, runId, agentId, providerConfig); const workerManager = createFreshPluginLeaseWorker(pluginId); const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId, heartbeatRunId: runId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, }); expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease"); expect(acquired.lease.leasePolicy).toBe("ephemeral"); expect(workerManager.call).toHaveBeenCalledTimes(1); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234); }); it("does not resume a lease when the nested capability disables reusable leases", async () => { // The provider declares the legacy `supportsReusableLeases: true` flag but // the nested `sandboxCapabilities.reusableLeases: false`. The nested value // wins through the capability contract, so acquisition must acquire a fresh // lease and must never resume the existing reusable lease. const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, }; const environment = { ...baseEnvironment, name: "Nested-disabled Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "acme.nested-disabled-sandbox-provider", packageName: "@acme/nested-disabled-sandbox-provider", displayName: "Nested-disabled Sandbox Provider", description: "Test provider with a legacy flag and a disabled nested capability", environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", supportsReusableLeases: true, sandboxCapabilities: { reusableLeases: false }, configSchema: { type: "object", properties: { image: { type: "string" }, timeoutMs: { type: "number" }, reuseLease: { type: "boolean" }, }, }, }, ], }); const executionWorkspaceId = randomUUID(); const projectId = randomUUID(); await insertReusableProject(projectId, companyId); await db.insert(executionWorkspaces).values({ id: executionWorkspaceId, companyId, projectId, mode: "shared_workspace", strategyType: "project_primary", name: "Nested-disabled workspace", status: "active", providerType: "local_fs", createdAt: new Date(), updatedAt: new Date(), }); await insertOldReusablePluginLease(companyId, environment, executionWorkspaceId, runId, agentId, providerConfig); const workerManager = createFreshPluginLeaseWorker(pluginId); const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId, heartbeatRunId: runId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, }); expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease"); expect(acquired.lease.leasePolicy).toBe("ephemeral"); expect(workerManager.call).toHaveBeenCalledTimes(1); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234); expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentResumeLease", expect.anything(), expect.anything(), ); }); it("fails closed and does not resume when the worker does not verify the reuse methods", async () => { // The provider declares `reusableLeases: true`, but its worker advertises // neither `environmentResumeLease` nor `environmentReleaseLease`. The runtime // must not resume or reuse the lease, because it cannot dispatch a resume or // a release the worker does not serve. It acquires a fresh ephemeral lease // and leaves the old reusable lease untouched. const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, }; const environment = { ...baseEnvironment, name: "Unverified-worker Plugin Sandbox", driver: "sandbox", config: providerConfig, }; await environmentService(db).update(environment.id, { driver: "sandbox", name: environment.name, config: providerConfig, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "acme.unverified-worker-sandbox-provider", packageName: "@acme/unverified-worker-sandbox-provider", displayName: "Unverified-worker Sandbox Provider", description: "Test provider that declares reusable leases but whose worker lacks the reuse methods", environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", sandboxCapabilities: { reusableLeases: true }, configSchema: { type: "object", properties: { image: { type: "string" }, timeoutMs: { type: "number" }, reuseLease: { type: "boolean" }, }, }, }, ], }); const executionWorkspaceId = randomUUID(); const projectId = randomUUID(); await insertReusableProject(projectId, companyId); await db.insert(executionWorkspaces).values({ id: executionWorkspaceId, companyId, projectId, mode: "shared_workspace", strategyType: "project_primary", name: "Unverified-worker workspace", status: "active", providerType: "local_fs", createdAt: new Date(), updatedAt: new Date(), }); const existingLease = await environmentService(db).acquireLease({ companyId, environmentId: environment.id, executionWorkspaceId, heartbeatRunId: runId, leasePolicy: "reuse_by_environment", provider: "fake-plugin", providerLeaseId: "old-plugin-lease", metadata: { agentId, provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, reusableSandboxLease: { version: 1, companyId, environmentId: environment.id, executionWorkspaceId, agentId, adapterType: null, provider: "fake-plugin", runtimeFingerprint: reusableRuntimeFingerprint({ provider: "fake-plugin", adapterType: null, config: providerConfig, }), }, }, }); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "fresh-plugin-lease", metadata: { provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true, remoteCwd: "/workspace", }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), // The worker verifies only `environmentExecute`. It advertises neither // reuse verb, so the reusable-lease capability fails closed. getWorker: vi.fn(() => ({ supportedMethods: ["environmentExecute"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId, heartbeatRunId: runId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, }); expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease"); expect(acquired.lease.leasePolicy).toBe("ephemeral"); expect(workerManager.call).toHaveBeenCalledTimes(1); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234); expect(workerManager.call).not.toHaveBeenCalledWith( pluginId, "environmentResumeLease", expect.anything(), expect.anything(), ); // The runtime leaves the old reusable lease untouched: it neither resumes // nor destroys a lease it cannot serve. await expect(environmentService(db).getLeaseById(existingLease.id)).resolves.toMatchObject({ status: "active", leasePolicy: "reuse_by_environment", }); }); it.each([ { status: "active", scope: "workspace" }, { status: "failed", scope: "workspace" }, { status: "failed", scope: "issue" }, ] as const)("destroys $status reusable plugin-backed sandbox leases scoped to an $scope", async ({ status, scope }) => { const { pluginId, companyId, runId, executionWorkspaceId, reusableLease } = await seedReusablePluginSandboxLease(); await db .update(heartbeatRuns) .set({ status: status === "failed" ? "failed" : "succeeded" }) .where(eq(heartbeatRuns.id, runId)); if (status === "failed") await environmentService(db).releaseLease(reusableLease.id, "failed"); const issueId = randomUUID(); if (scope === "issue") { await db.insert(issues).values({ id: issueId, companyId, title: "Failed reusable lease cleanup", status: "done", priority: "medium" }); await db.update(environmentLeases).set({ issueId }).where(eq(environmentLeases.id, reusableLease.id)); } const selection = scope === "issue" ? { issueId } : { executionWorkspaceId }; const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentDestroyLease") { return undefined; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); // A known issue/workspace id does not authorize another company's teardown. expect(await runtimeWithPlugin.destroyReusableSandboxLeases({ companyId: randomUUID(), ...selection })).toEqual([]); expect(workerManager.call).not.toHaveBeenCalled(); const destroyed = await runtimeWithPlugin.destroyReusableSandboxLeases({ companyId, ...selection, failureReason: "execution_workspace_closed", }); expect(destroyed).toHaveLength(1); expect(destroyed[0]?.lease.id).toBe(reusableLease.id); expect(destroyed[0]?.lease.status).toBe("expired"); expect(workerManager.call).toHaveBeenCalledWith( pluginId, "environmentDestroyLease", expect.objectContaining({ driverKey: "fake-plugin", providerLeaseId: "reusable-plugin-lease", }), 31234, ); await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ status: "expired", failureReason: "execution_workspace_closed", cleanupStatus: "success", }); }); it.each(["active", "failed"] as const)("does not destroy a %s scoped reusable lease while its run is active", async (status) => { const { pluginId, companyId, executionWorkspaceId, reusableLease } = await seedReusablePluginSandboxLease(); if (status === "failed") await environmentService(db).releaseLease(reusableLease.id, "failed"); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async () => undefined), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager, }); const destroyed = await runtimeWithPlugin.destroyReusableSandboxLeases({ companyId, executionWorkspaceId, failureReason: "issue_terminal_done", }); expect(destroyed).toEqual([]); expect(workerManager.call).not.toHaveBeenCalled(); await expect( environmentService(db).getLeaseById(reusableLease.id), ).resolves.toMatchObject({ status }); }); it.each(["issue", "workspace", "environment"] as const)( "durably claims failed reusable cleanup before %s teardown and recovers a provider failure", async (scope) => { const seeded = await seedReusablePluginSandboxLease(); await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId)); await environmentService(db).releaseLease(seeded.reusableLease.id, "failed"); const issueId = randomUUID(); if (scope === "issue") { await db.insert(issues).values({ id: issueId, companyId: seeded.companyId, title: "Cleanup crash window", status: "done" }); await db.update(environmentLeases).set({ issueId }).where(eq(environmentLeases.id, seeded.reusableLease.id)); } let enterProvider!: () => void; let finishProvider!: () => void; const entered = new Promise((resolve) => { enterProvider = resolve; }); const finish = new Promise((resolve) => { finishProvider = resolve; }); let unavailable = true; const call = vi.fn(async (_id: string, method: string) => { if (method !== "environmentDestroyLease") throw new Error(`Unexpected method ${method}`); if (unavailable) { enterProvider(); await finish; throw new Error("Provider response lost"); } return { providerLeaseId: seeded.reusableLease.providerLeaseId, state: "destroyed" }; }); const makeRuntime = () => environmentRuntimeService(db, { pluginWorkerManager: { isRunning: () => true, call, getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }), } as unknown as PluginWorkerManager }); const runtime = makeRuntime(); const close = (service: ReturnType) => scope === "environment" ? service.destroyReusableSandboxLeasesForEnvironment({ environmentId: seeded.environment.id }) : service.destroyReusableSandboxLeases({ companyId: seeded.companyId, ...(scope === "issue" ? { issueId } : { executionWorkspaceId: seeded.executionWorkspaceId }) }); const closing = close(runtime); try { await entered; // This independently persisted row is what survives controller loss // during the provider call. No provider receipt has arrived yet. expect(await environmentService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({ status: "pending_cleanup", cleanupStatus: "failed", companyId: seeded.companyId, providerLeaseId: seeded.reusableLease.providerLeaseId, metadata: { pendingCleanupAttemptId: expect.any(String), pendingCleanupInFlight: true, pendingCleanupLeaseExpiresAtMs: expect.any(Number) }, }); await close(makeRuntime()); expect(await heartbeatService(db, { environmentRuntime: makeRuntime() }).sweepPendingCleanupLeases({ backoffMs: 0 })) .toEqual({ swept: 0, destroyed: 0, capped: 0 }); expect(call).toHaveBeenCalledOnce(); } finally { finishProvider(); await closing; } expect(await environmentService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({ status: "pending_cleanup", metadata: { pendingCleanupInFlight: false }, }); unavailable = false; expect(await heartbeatService(db, { environmentRuntime: makeRuntime() }).sweepPendingCleanupLeases({ backoffMs: 0 })) .toEqual({ swept: 1, destroyed: 1, capped: 0 }); expect(call).toHaveBeenCalledTimes(2); expect(await environmentService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({ status: "expired", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "destroyed" } }, }); }, ); it.each(["queued", "scheduled_retry", "running"] as const)( "fences scoped cleanup when its holding run becomes %s after selection", async (status) => { const seeded = await seedReusablePluginSandboxLease(); await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId)); await environmentService(db).releaseLease(seeded.reusableLease.id, "failed"); const originalService = environmentsModule.environmentService; const serviceSpy = vi.spyOn(environmentsModule, "environmentService").mockImplementation((client) => { const service = originalService(client); return { ...service, getById: async (id: string) => { const environment = await service.getById(id); if (id === seeded.environment.id) { await db.update(heartbeatRuns).set({ status }).where(eq(heartbeatRuns.id, seeded.runId)); } return environment; } }; }); const call = vi.fn(async () => undefined); const runtime = environmentRuntimeService(db, { pluginWorkerManager: { isRunning: () => true, call, getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }), } as unknown as PluginWorkerManager }); try { expect(await runtime.destroyReusableSandboxLeases({ companyId: seeded.companyId, executionWorkspaceId: seeded.executionWorkspaceId })).toEqual([]); expect(call).not.toHaveBeenCalled(); expect(await originalService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({ status: "failed" }); } finally { serviceSpy.mockRestore(); } }, ); it("destroys reusable plugin-backed sandbox leases scoped to an environment", async () => { const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease(); // The holding run is finished, so the reservation is stale and destroyable. await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, runId)); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentDestroyLease") { return { providerLeaseId: reusableLease.providerLeaseId, state: "destroyed" }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const result = await runtimeWithPlugin.destroyReusableSandboxLeasesForEnvironment({ environmentId: reusableLease.environmentId!, failureReason: "environment_deleted", }); expect(result).toEqual({ destroyed: 1, failed: 0, skippedLiveRun: 0 }); expect(workerManager.call).toHaveBeenCalledWith( pluginId, "environmentDestroyLease", expect.objectContaining({ driverKey: "fake-plugin", providerLeaseId: "reusable-plugin-lease", }), 31234, ); await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ status: "expired", failureReason: "environment_deleted", cleanupStatus: "success", metadata: { remoteExecutionTermination: { schema: "paperclip.remote-termination.v1", leaseId: reusableLease.id, runId, providerLeaseId: reusableLease.providerLeaseId, state: "destroyed" } }, }); }); it("destroys a failed reusable sandbox after successful release before deleting its environment", async () => { const { pluginId, companyId, runId, environment, reusableLease } = await seedReusablePluginSandboxLease(); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentReleaseLease") { return { providerLeaseId: reusableLease.providerLeaseId, state: "stopped" }; } if (method === "environmentDestroyLease") { // The provider call must already have a durable cleanup reference. expect(await environmentService(db).getLeaseById(reusableLease.id)).toMatchObject({ status: "pending_cleanup", environmentId: environment.id, }); return { providerLeaseId: reusableLease.providerLeaseId, state: "destroyed" }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, runId)); const released = await runtimeWithPlugin.releaseRunLeases(runId, "failed", undefined, undefined, true); expect(released[0]?.lease).toMatchObject({ id: reusableLease.id, status: "failed", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "stopped" } }, }); const result = await runtimeWithPlugin.destroyReusableSandboxLeasesForEnvironment({ environmentId: environment.id, failureReason: "environment_deleted", }); expect(result).toEqual({ destroyed: 1, failed: 0, skippedLiveRun: 0 }); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease", expect.objectContaining({ companyId, environmentId: environment.id, providerLeaseId: reusableLease.providerLeaseId }), 31234); await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ status: "expired", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "destroyed" } }, }); expect((await environmentService(db).removeIfDeletable(environment.id))?.id).toBe(environment.id); }); it.each(["active", "failed"] as const)("keeps a %s reusable lease held by an in-flight run out of the environment-scoped destroy", async (status) => { const { pluginId, reusableLease } = await seedReusablePluginSandboxLease(); // seedEnvironment leaves the holding run in `running` status. if (status === "failed") await environmentService(db).releaseLease(reusableLease.id, "failed"); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async () => undefined), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const result = await runtimeWithPlugin.destroyReusableSandboxLeasesForEnvironment({ environmentId: reusableLease.environmentId!, failureReason: "environment_deleted", }); expect(result).toEqual({ destroyed: 0, failed: 0, skippedLiveRun: 1 }); expect(workerManager.call).not.toHaveBeenCalled(); // The lease keeps its reusable status, so the delete guard still blocks. await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ status, leasePolicy: "reuse_by_environment", }); }); it("sweeps reusable cleanup with the configuration recorded on the lease", async () => { const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease(); const environmentsSvc = environmentService(db); await environmentsSvc.releaseLease(reusableLease.id, "pending_cleanup", { failureReason: "release_cleanup_failed", cleanupStatus: "failed", }); // Re-point the environment after the reusable sandbox was provisioned. // The pending-cleanup sweep still enters destroyRunLease because the // environment exists, but destroy must target the provider and config // captured on the lease rather than this current configuration. await environmentsSvc.update(environment.id, { config: { provider: "replacement-provider", image: "replacement:test", timeoutMs: 9999, reuseLease: true, }, }); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentDestroyLease") return undefined; throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const heartbeat = heartbeatService(db, { environmentRuntime: runtimeWithPlugin }); const result = await heartbeat.sweepPendingCleanupLeases({ backoffMs: 0 }); expect(result).toEqual({ swept: 1, destroyed: 1, capped: 0 }); expect(workerManager.call).toHaveBeenCalledWith( pluginId, "environmentDestroyLease", expect.objectContaining({ driverKey: "fake-plugin", environmentId: environment.id, providerLeaseId: "reusable-plugin-lease", config: expect.objectContaining({ image: "fake:test", timeoutMs: 1234, reuseLease: true, }), }), 31234, ); const destroyInput = vi.mocked(workerManager.call).mock.calls[0]?.[2] as { config?: Record; }; expect(destroyInput.config).not.toMatchObject({ image: "replacement:test", timeoutMs: 9999, }); await expect(environmentsSvc.getLeaseById(reusableLease.id)).resolves.toMatchObject({ status: "expired", cleanupStatus: "success", }); }); it("retries reusable plugin-backed sandbox destroy when the worker is unavailable", async () => { const { pluginId, companyId, executionWorkspaceId, runId, reusableLease } = await seedReusablePluginSandboxLease(); await db .update(heartbeatRuns) .set({ status: "succeeded", finishedAt: new Date() }) .where(eq(heartbeatRuns.id, runId)); const offlineWorkerManager = { isRunning: vi.fn(() => false), call: vi.fn(), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithOfflinePlugin = environmentRuntimeService(db, { pluginWorkerManager: offlineWorkerManager, }); const pending = await runtimeWithOfflinePlugin.destroyReusableSandboxLeases({ companyId, executionWorkspaceId, failureReason: "execution_workspace_closed", }); expect(pending).toHaveLength(1); expect(pending[0]?.lease.id).toBe(reusableLease.id); expect(pending[0]?.lease.status).toBe("pending_cleanup"); expect(offlineWorkerManager.call).not.toHaveBeenCalled(); await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ status: "pending_cleanup", failureReason: "execution_workspace_closed", cleanupStatus: "failed", }); const recoveredWorkerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentDestroyLease") { return undefined; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithRecoveredPlugin = environmentRuntimeService(db, { pluginWorkerManager: recoveredWorkerManager, }); const retried = await runtimeWithRecoveredPlugin.destroyReusableSandboxLeases({ companyId, executionWorkspaceId, failureReason: "cleanup_retry", }); expect(retried).toHaveLength(1); expect(retried[0]?.lease.id).toBe(reusableLease.id); expect(retried[0]?.lease.status).toBe("expired"); expect(recoveredWorkerManager.call).toHaveBeenCalledWith( pluginId, "environmentDestroyLease", expect.objectContaining({ driverKey: "fake-plugin", providerLeaseId: "reusable-plugin-lease", }), 31234, ); await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({ status: "expired", failureReason: "cleanup_retry", cleanupStatus: "success", }); }); it("resolves effective capabilities from the lease's exact plugin, not an earlier plugin that shares the driver key", async () => { // The helper seeds the plugin that owns the lease. Pin it to `pluginId` // through the lease metadata and give it a lower-priority sibling. const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease(); // Rewrite the owner plugin so it DENIES reusable leases through the nested // capability declaration. await db .update(plugins) .set({ manifestJson: { id: "acme.reusable-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Reusable Sandbox Provider", description: "Owner plugin that denies reusable leases", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", sandboxCapabilities: { reusableLeases: false }, configSchema: { type: "object", properties: {} }, }, ], }, updatedAt: new Date(), } as any) .where(eq(plugins.id, pluginId)); // Install an EARLIER plugin that shares the driver key and grants reusable // leases. It is not ready and never acquired this lease. A resolver keyed by // driver key alone would read this declaration and grant a capability the // owner plugin denied. const collidingPluginId = randomUUID(); await insertSandboxProviderPlugin(collidingPluginId, { pluginKey: "acme.colliding-sandbox-provider", packageName: "@acme/colliding-sandbox-provider", displayName: "Colliding Sandbox Provider", description: "Earlier plugin that shares the driver key", environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", supportsReusableLeases: true, configSchema: { type: "object", properties: {} }, }, ], status: "installed", installOrder: 0, }); // The owner worker verifies the reuse verbs and the sync verbs, so every // capability is verified. Only the owner's declaration can narrow one. const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(), getWorker: vi.fn((id: string) => id === pluginId ? { supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentSyncIn", "environmentSyncOut", ], } : undefined, ), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); expect(reusableLease.metadata?.pluginId).toBe(pluginId); const effective = await runtimeWithPlugin.resolveCapabilities({ environment, lease: reusableLease, }); // The runtime read the owner plugin's declaration, so it denies reusable // leases even though the earlier plugin grants them. expect(effective?.reusableLeases).toBe(false); // The owner plugin does not restrict native sync, and its worker verifies // the sync verbs, so those stay granted. This proves the resolver read the // owner declaration and did not fail every capability closed. expect(effective?.nativeSyncIn).toBe(true); expect(effective?.nativeSyncOut).toBe(true); }); it("fails every effective capability closed when the pinned plugin id is absent from the registry", async () => { // The lease pins a plugin id, but that plugin record is gone. A stale worker // entry still advertises every method. The runtime must not read the stale // methods; it must fail closed because the exact-plugin identity is gone. const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease(); await db.delete(plugins).where(eq(plugins.id, pluginId)); const workerManager = { isRunning: vi.fn(() => true), call: vi.fn(), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentExecute", "environmentSyncIn", "environmentSyncOut", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const effective = await runtimeWithPlugin.resolveCapabilities({ environment, lease: reusableLease, }); for (const key of SANDBOX_CAPABILITY_KEYS) { expect(effective?.[key]).toBe(false); } }); it("fails every effective capability closed when the pinned plugin no longer declares this provider key", async () => { // The pinned plugin still exists, but it no longer declares a // `sandbox_provider` driver with this key (here it changed the driver kind). // A running worker still advertises every method. The runtime must fail // closed because the exact-plugin declaration is gone. const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease(); await db .update(plugins) .set({ manifestJson: { id: "acme.reusable-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Reusable Sandbox Provider", description: "Owner plugin that no longer declares the provider key", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", // The key exists, but the kind is now a plain environment driver, // not a sandbox provider. The by-id resolver fails closed. kind: "environment_driver", displayName: "Fake Plugin", configSchema: { type: "object", properties: {} }, }, ], }, updatedAt: new Date(), } as any) .where(eq(plugins.id, pluginId)); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", "environmentExecute", "environmentSyncIn", "environmentSyncOut", ], })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const effective = await runtimeWithPlugin.resolveCapabilities({ environment, lease: reusableLease, }); for (const key of SANDBOX_CAPABILITY_KEYS) { expect(effective?.[key]).toBe(false); } }); it("defers to verified worker discovery for a valid pinned plugin that omits sandboxCapabilities", async () => { // A valid, identified plugin whose manifest declares no `sandboxCapabilities` // and no legacy reuse flag. Its worker verifies the sync verbs. An omitted // declaration is NOT an identity failure: the runtime defers to the verified // baseline, so native sync stays granted while unverified capabilities stay // false. const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease(); await db .update(plugins) .set({ manifestJson: { id: "acme.reusable-sandbox-provider", apiVersion: 1, version: "1.0.0", displayName: "Reusable Sandbox Provider", description: "Owner plugin that omits the capability declaration", author: "Paperclip", categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" }, environmentDrivers: [ { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", configSchema: { type: "object", properties: {} }, }, ], }, updatedAt: new Date(), } as any) .where(eq(plugins.id, pluginId)); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(), getWorker: vi.fn((id: string) => id === pluginId ? { supportedMethods: ["environmentSyncIn", "environmentSyncOut"] } : undefined, ), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const effective = await runtimeWithPlugin.resolveCapabilities({ environment, lease: reusableLease, }); // The worker verified the sync verbs and the omitted declaration adds no // restriction, so native sync stays granted. expect(effective?.nativeSyncIn).toBe(true); expect(effective?.nativeSyncOut).toBe(true); // The worker did not verify the reuse verbs, so reusable leases stay false. expect(effective?.reusableLeases).toBe(false); }); it("releases a sandbox run lease from metadata after the environment config changes", async () => { const { companyId, environment, runId } = await seedEnvironment({ driver: "sandbox", name: "Fake Sandbox", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: true, }, }); const acquired = await runtime.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); await environmentService(db).update(environment.id, { driver: "local", config: {}, }); const released = await runtime.releaseRunLeases(runId); expect(released).toHaveLength(1); expect(released[0]?.lease.id).toBe(acquired.lease.id); expect(released[0]?.lease.status).toBe("released"); }); it("does not reuse a sandbox lease owned by a different agent for the same execution workspace", async () => { const { companyId, agentId, environment, runId } = await seedEnvironment({ driver: "plugin", name: "Plugin Fake plugin", config: { pluginKey: "acme.environments", driverKey: "fake-plugin", driverConfig: { template: "base", }, }, }); const otherAgentId = randomUUID(); const executionWorkspaceId = randomUUID(); const pluginId = randomUUID(); const projectId = randomUUID(); await insertReusableProject(projectId, companyId); await db.insert(executionWorkspaces).values({ id: executionWorkspaceId, companyId, projectId, mode: "shared_workspace", strategyType: "project_primary", name: "Existing workspace", status: "active", providerType: "local_fs", createdAt: new Date(), updatedAt: new Date(), }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "acme.environments", packageName: "@acme/paperclip-environments", displayName: "Acme Environments", description: "Test plugin environment driver", author: "Acme", }); await environmentService(db).acquireLease({ companyId, environmentId: environment.id, executionWorkspaceId, heartbeatRunId: runId, leasePolicy: "reuse_by_environment", provider: "fake-plugin", providerLeaseId: "other-agent-lease", metadata: { agentId, provider: "fake-plugin", template: "base", reuseLease: true, }, }); const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "fresh-agent-lease", metadata: { provider: "fake-plugin", template: "base", reuseLease: true, }, }; } throw new Error(`Unexpected plugin method: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, agentId: otherAgentId, heartbeatRunId: runId, persistedExecutionWorkspace: { id: executionWorkspaceId, mode: "shared_workspace", }, }); expect(acquired.lease.providerLeaseId).toBe("fresh-agent-lease"); expect(workerManager.call).toHaveBeenCalledTimes(1); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.objectContaining({ agentId: otherAgentId, executionWorkspaceId, })); }); it.each([undefined, 300_000])("delegates plugin environment leases with acquisition budget %s", async (defaultAcquireTimeoutMs) => { const pluginId = randomUUID(); const expiresAt = new Date(Date.now() + 60_000).toISOString(); const workerManager = { isRunning: vi.fn(() => true), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "plugin-lease-1", expiresAt, metadata: { driver: "local", pluginId: "provider-plugin-id", pluginKey: "provider.plugin", driverKey: "provider-driver", executionWorkspaceMode: "provider-mode", provider: "test-provider", remoteCwd: "/workspace", }, }; } return undefined; }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager, }); const { companyId, environment, runId } = await seedEnvironment({ driver: "plugin", name: "Plugin Fake plugin", config: { pluginKey: "acme.environments", driverKey: "fake-plugin", driverConfig: { template: "base", }, }, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "acme.environments", packageName: "@acme/paperclip-environments", displayName: "Acme Environments", description: "Test plugin environment driver", author: "Acme", environmentDrivers: [ { driverKey: "fake-plugin", displayName: "Fake plugin", defaultAcquireTimeoutMs, configSchema: { type: "object" }, }, ], }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", { driverKey: "fake-plugin", companyId, environmentId: environment.id, executionWorkspaceId: undefined, executionWorkspaceSettings: null, issueId: null, config: { template: "base" }, agentId: undefined, adapterType: undefined, runId, workspaceMode: undefined, }, ...(defaultAcquireTimeoutMs === undefined ? [] : [330_000])); expect(acquired.lease.providerLeaseId).toBe("plugin-lease-1"); expect(acquired.lease.expiresAt?.toISOString()).toBe(expiresAt); expect(acquired.lease.metadata).toMatchObject({ driver: "plugin", pluginId, pluginKey: "acme.environments", driverKey: "fake-plugin", executionWorkspaceMode: null, providerMetadata: { driver: "local", pluginId: "provider-plugin-id", pluginKey: "provider.plugin", driverKey: "provider-driver", executionWorkspaceMode: "provider-mode", provider: "test-provider", remoteCwd: "/workspace", }, }); await environmentService(db).update(environment.id, { driver: "local", config: {}, }); const released = await runtimeWithPlugin.releaseRunLeases(runId); expect(released).toHaveLength(1); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentReleaseLease", { driverKey: "fake-plugin", companyId, environmentId: environment.id, issueId: null, config: {}, providerLeaseId: "plugin-lease-1", leaseMetadata: expect.objectContaining({ driver: "plugin", pluginId, providerMetadata: expect.objectContaining({ driver: "local", }), }), }); expect(released[0]?.lease.status).toBe("released"); }); async function seedPluginDriverEnvironment(pluginId: string) { const seeded = await seedEnvironment({ driver: "plugin", name: "Plugin Fake plugin", config: { pluginKey: "acme.environments", driverKey: "fake-plugin", driverConfig: { template: "base" }, }, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "acme.environments", packageName: "@acme/paperclip-environments", displayName: "Acme Environments", description: "Test plugin environment driver", author: "Acme", environmentDrivers: [ { driverKey: "fake-plugin", displayName: "Fake plugin", configSchema: { type: "object" } }, ], }); return seeded; } it("forwards a requested expiry to the acquire RPC and records only a provider-attested expiry", async () => { const pluginId = randomUUID(); let receivedRequestedExpiresAt: unknown; const workerManager = { isRunning: vi.fn(() => true), call: vi.fn(async (_pluginId: string, method: string, params: Record) => { if (method === "environmentAcquireLease") { receivedRequestedExpiresAt = params.requestedExpiresAt; // The provider grants no expiry. return { providerLeaseId: "plugin-lease-no-expiry", metadata: { remoteCwd: "/workspace" } }; } return undefined; }), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const { companyId, environment, runId } = await seedPluginDriverEnvironment(pluginId); const requestedExpiresAt = new Date(Date.now() + 60_000); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, requestedExpiresAt, }); // The acquire RPC receives the requested expiry as an ISO 8601 string. expect(receivedRequestedExpiresAt).toBe(requestedExpiresAt.toISOString()); // The provider gave no expiry, so the lease row records no expiry. The runtime // never synthesizes the requested deadline as evidence of provider enforcement. expect(acquired.lease.expiresAt ?? null).toBeNull(); }); it("records the real provider expiry when the provider grants one later than the requested deadline", async () => { const pluginId = randomUUID(); const providerExpiresAt = new Date(Date.now() + 120_000).toISOString(); const workerManager = { isRunning: vi.fn(() => true), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "plugin-lease-later", expiresAt: providerExpiresAt, metadata: { remoteCwd: "/workspace" }, }; } return undefined; }), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); const { companyId, environment, runId } = await seedPluginDriverEnvironment(pluginId); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, requestedExpiresAt: new Date(Date.now() + 60_000), }); // The lease row records the real provider expiry, not the requested deadline. // The caller then verifies the expiry bounds its deadline and fails closed. expect(acquired.lease.expiresAt?.toISOString()).toBe(providerExpiresAt); }); it("delegates the full plugin environment lifecycle through the worker manager", async () => { const pluginId = randomUUID(); const workerManager = { isRunning: vi.fn(() => true), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentAcquireLease") { return { providerLeaseId: "plugin-lease-full", metadata: { remoteCwd: "/workspace", }, }; } if (method === "environmentResumeLease") { return { providerLeaseId: "plugin-lease-full", metadata: { resumed: true, }, }; } if (method === "environmentRealizeWorkspace") { return { cwd: "/workspace/project", metadata: { realized: true, }, }; } if (method === "environmentExecute") { return { exitCode: 0, signal: null, timedOut: false, stdout: "ok\n", stderr: "", metadata: { commandId: "cmd-1", }, }; } return undefined; }), getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager, }); const { companyId, environment, runId } = await seedEnvironment({ driver: "plugin", name: "Plugin Full Lifecycle", config: { pluginKey: "acme.environments", driverKey: "fake-plugin", driverConfig: { template: "base", }, }, }); await insertSandboxProviderPlugin(pluginId, { pluginKey: "acme.environments", packageName: "@acme/paperclip-environments", displayName: "Acme Environments", description: "Test plugin environment driver", author: "Acme", environmentDrivers: [ { driverKey: "fake-plugin", displayName: "Fake plugin", configSchema: { type: "object" }, }, ], }); const acquired = await runtimeWithPlugin.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); const resumed = await runtimeWithPlugin.resumeRunLease({ environment, lease: acquired.lease, }); const realized = await runtimeWithPlugin.realizeWorkspace({ environment, lease: acquired.lease, workspace: { localPath: "/tmp/project", mode: "ephemeral", }, }); const executed = await runtimeWithPlugin.execute({ environment, lease: acquired.lease, command: "echo", args: ["ok"], cwd: realized.cwd, env: { FOO: "bar" }, stdin: "", timeoutMs: 1000, }); const destroyed = await runtimeWithPlugin.destroyRunLease({ environment, lease: acquired.lease, }); expect(resumed).toMatchObject({ providerLeaseId: "plugin-lease-full", metadata: { resumed: true, }, }); expect(realized).toEqual({ cwd: "/workspace/project", metadata: { realized: true, }, }); expect(executed).toMatchObject({ exitCode: 0, timedOut: false, stdout: "ok\n", }); expect(destroyed?.status).toBe("failed"); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentResumeLease", { driverKey: "fake-plugin", companyId, environmentId: environment.id, issueId: null, config: { template: "base" }, providerLeaseId: "plugin-lease-full", leaseMetadata: expect.objectContaining({ driver: "plugin", pluginId, }), }); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentRealizeWorkspace", expect.objectContaining({ driverKey: "fake-plugin", companyId, environmentId: environment.id, config: { template: "base" }, workspace: { localPath: "/tmp/project", mode: "ephemeral", }, })); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentExecute", expect.objectContaining({ driverKey: "fake-plugin", companyId, environmentId: environment.id, command: "echo", args: ["ok"], cwd: "/workspace/project", env: { FOO: "bar" }, }), 31000); expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease", { driverKey: "fake-plugin", companyId, environmentId: environment.id, issueId: null, config: { template: "base" }, providerLeaseId: "plugin-lease-full", leaseMetadata: expect.objectContaining({ driver: "plugin", pluginId, }), }); }); it("releases with the driver captured on the lease even if the environment driver changes later", async () => { const { companyId, environment, runId } = await seedEnvironment(); const environmentsSvc = environmentService(db); const localRelease = vi.fn(async ({ lease, status }: { lease: { id: string }; status: "released" | "expired" | "failed" }) => await environmentsSvc.releaseLease(lease.id, status) ); const sshRelease = vi.fn(async () => { throw new Error("ssh release should not be called"); }); const runtimeWithSpies = environmentRuntimeService(db, { drivers: [ { driver: "local", acquireRunLease: async (input) => await environmentsSvc.acquireLease({ companyId: input.companyId, environmentId: input.environment.id, executionWorkspaceId: input.executionWorkspaceId, issueId: input.issueId, heartbeatRunId: input.heartbeatRunId, metadata: { driver: input.environment.driver, executionWorkspaceMode: input.executionWorkspaceMode, }, }), releaseRunLease: localRelease, }, { driver: "ssh", acquireRunLease: async () => { throw new Error("ssh acquire should not be called"); }, releaseRunLease: sshRelease, }, ], }); const acquired = await runtimeWithSpies.acquireRunLease({ companyId, environment, issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, }); await environmentsSvc.update(environment.id, { driver: "ssh" }); const released = await runtimeWithSpies.releaseRunLeases(runId); expect(released).toHaveLength(1); expect(localRelease).toHaveBeenCalledTimes(1); expect(sshRelease).not.toHaveBeenCalled(); expect(acquired.lease.metadata?.driver).toBe("local"); }); it("test_release_run_leases_continues_after_first_release_fails", async () => { const { companyId, environment, runId } = await seedEnvironment(); const environmentsSvc = environmentService(db); // Seed two active leases for one run. The first driver release throws; the // second must still release. const failingLease = await environmentsSvc.acquireLease({ companyId, environmentId: environment.id, heartbeatRunId: runId, provider: "local", providerLeaseId: "fail-release", metadata: { driver: "local" }, }); const healthyLease = await environmentsSvc.acquireLease({ companyId, environmentId: environment.id, heartbeatRunId: runId, provider: "local", providerLeaseId: "healthy-release", metadata: { driver: "local" }, }); const runtimeWithFailingDriver = environmentRuntimeService(db, { drivers: [ { driver: "local", acquireRunLease: async () => { throw new Error("acquire should not be called"); }, releaseRunLease: async ({ lease, status }) => { if (lease.providerLeaseId === "fail-release") { throw new Error("driver release failed"); } return await environmentsSvc.releaseLease(lease.id, status); }, }, ], }); const errors: Array<{ leaseId: string; error: unknown }> = []; const released = await runtimeWithFailingDriver.releaseRunLeases( runId, "released", (leaseId, error) => errors.push({ leaseId, error }), ); // The healthy lease released even though the first release failed. expect(released).toHaveLength(1); expect(released[0]?.lease.id).toBe(healthyLease.id); expect(released[0]?.lease.status).toBe("released"); // The failed release reported one lease-specific error. expect(errors).toHaveLength(1); expect(errors[0]?.leaseId).toBe(failingLease.id); expect((errors[0]?.error as Error).message).toBe("driver release failed"); // The database confirms the isolation. The healthy lease released; the // failing lease stayed active. const rows = await db.select().from(environmentLeases); expect(rows.find((row) => row.id === healthyLease.id)?.status).toBe("released"); expect(rows.find((row) => row.id === failingLease.id)?.status).toBe("active"); }); });