PaperClipAI/server/src/__tests__/environment-runtime.test.ts

import { JsonRpcCallError } from "@paperclipai/plugin-sdk";
import { collectRunFailureDiagnostics } from "../services/run-failure-diagnostics.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
import { createHash, randomUUID } from "node:crypto";
import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { eq } from "drizzle-orm";
import {
  buildSshEnvLabFixtureConfig,
  getSshEnvLabSupport,
  startSshEnvLabFixture,
  stopSshEnvLabFixture,
} from "@paperclipai/adapter-utils/ssh";
import {
  activityLog,
  costEvents,
  agents,
  builtInManagedResources,
  companies,
  companySecretVersions,
  companySecrets,
  createDb,
  environmentLeases,
  environments,
  executionWorkspaces,
  heartbeatRuns,
  issues,
  plugins,
  projects,
} from "@paperclipai/db";
import {
  getEmbeddedPostgresTestSupport,
  startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
import { resolveEnvironmentDriverConfigForRuntime } from "../services/environment-config.ts";
import {
  SANDBOX_CAPABILITY_KEYS,
  environmentRuntimeService,
  findReusableSandboxLeaseId,
  SandboxOrphanCleanupWriteError,
} from "../services/environment-runtime.ts";
import * as sandboxProviderRuntime from "../services/sandbox-provider-runtime.ts";
import * as environmentsModule from "../services/environments.ts";
import { logger } from "../middleware/logger.ts";
import { resolveRunnerEnvironmentForRun } from "../services/runner-environment-lifecycle.js";
import { environmentService } from "../services/environments.ts";
import { remoteExecutionHasStopped } from "../services/remote-execution-termination.ts";
import { heartbeatService } from "../services/heartbeat.ts";
import { secretService } from "../services/secrets.ts";
import type { PluginWorkerManager } from "../services/plugin-worker-manager.ts";
import {
  getActiveStepContext,
  runWithRuntimeParent,
  type StartupSpanContext,
} from "@paperclipai/adapter-utils/acpx-engine/startup-timing";
import { traceparentFromContextToken } from "../instrumentation.ts";
import { ROOT_CONTEXT, trace } from "@opentelemetry/api";
import { buildNativeHarnessBackupManifest } from "../services/native-runtime/native-session-executor.ts";
import { createNativeHarnessBackupStamp } from "../services/native-runtime/native-harness-backup-stamp.ts";

const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
const sshFixtureSupport = await getSshEnvLabSupport();

if (!embeddedPostgresSupport.supported) {
  console.warn(
    `Skipping embedded Postgres environment runtime tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
  );
}

function stableStringify(value: unknown): string {
  if (Array.isArray(value)) {
    return `[${value.map((entry) => stableStringify(entry)).join(",")}]`;
  }
  if (value && typeof value === "object") {
    const record = value as Record<string, unknown>;
    return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`).join(",")}}`;
  }
  return JSON.stringify(value) ?? "null";
}

function reusableRuntimeFingerprint(input: {
  provider: string;
  adapterType: string | null;
  config: Record<string, unknown>;
}) {
  return createHash("sha256").update(stableStringify(input)).digest("hex");
}

describe("findReusableSandboxLeaseId", () => {
  it("matches reusable plugin-backed sandbox leases by provider", () => {
    const selected = findReusableSandboxLeaseId({
      config: {
        provider: "fake-plugin",
        image: "template-b",
        timeoutMs: 300000,
        reuseLease: true,
      },
      leases: [
        {
          providerLeaseId: "sandbox-template-a",
          metadata: {
            provider: "fake-plugin",
            image: "template-a",
            timeoutMs: 300000,
            reuseLease: true,
          },
        },
        {
          providerLeaseId: "sandbox-template-b",
          metadata: {
            provider: "fake-plugin",
            image: "template-b",
            timeoutMs: 300000,
            reuseLease: true,
          },
        },
      ],
    });

    expect(selected).toBe("sandbox-template-b");
  });

  it("ignores host-only log streaming when matching a reusable plugin lease", () => {
    const selected = findReusableSandboxLeaseId({
      config: {
        provider: "fake-plugin",
        image: "template-b",
        timeoutMs: 300000,
        reuseLease: true,
        streamRunLogs: true,
        runnerLifecycleMode: "warm",
        target: null,
      },
      leases: [
        {
          providerLeaseId: "sandbox-template-b",
          metadata: {
            provider: "fake-plugin",
            image: "template-b",
            timeoutMs: 300000,
            reuseLease: true,
            runnerLifecycleMode: "warm",
            target: "us",
          },
        },
      ],
    });

    expect(selected).toBe("sandbox-template-b");
  });

  it("requires image identity for reusable fake sandbox leases", () => {
    const selected = findReusableSandboxLeaseId({
      config: {
        provider: "fake",
        image: "ubuntu:24.04",
        reuseLease: true,
      },
      leases: [
        {
          providerLeaseId: "sandbox-image-a",
          metadata: {
            provider: "fake",
            image: "debian:12",
            reuseLease: true,
          },
        },
        {
          providerLeaseId: "sandbox-image-b",
          metadata: {
            provider: "fake",
            image: "ubuntu:24.04",
            reuseLease: true,
          },
        },
      ],
    });

    expect(selected).toBe("sandbox-image-b");
  });
});

describeEmbeddedPostgres("environmentRuntimeService", () => {
  let stopDb: (() => Promise<void>) | null = null;
  let db!: ReturnType<typeof createDb>;
  let runtime!: ReturnType<typeof environmentRuntimeService>;
  const fixtureRoots: string[] = [];
  // Give each test its own orphan-cleanup spool directory, so a spool file one
  // test writes never leaks into another test's flush. The default runtime reads
  // this env override when a test does not pass an explicit spool directory.
  let orphanCleanupSpoolDir: string | null = null;
  const previousOrphanCleanupSpoolDir = process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR;

  beforeAll(async () => {
    const started = await startEmbeddedPostgresTestDatabase("environment-runtime");
    stopDb = started.stop;
    db = createDb(started.connectionString);
    runtime = environmentRuntimeService(db);
  });

  beforeEach(async () => {
    orphanCleanupSpoolDir = await mkdtemp(path.join(os.tmpdir(), "orphan-cleanup-spool-"));
    process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR = orphanCleanupSpoolDir;
  });

  afterEach(async () => {
    if (orphanCleanupSpoolDir) {
      await rm(orphanCleanupSpoolDir, { recursive: true, force: true }).catch(() => undefined);
      orphanCleanupSpoolDir = null;
    }
    if (previousOrphanCleanupSpoolDir === undefined) {
      delete process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR;
    } else {
      process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR = previousOrphanCleanupSpoolDir;
    }
    while (fixtureRoots.length > 0) {
      const root = fixtureRoots.pop();
      if (!root) continue;
      await stopSshEnvLabFixture(path.join(root, "state.json")).catch(() => undefined);
      await rm(root, { recursive: true, force: true }).catch(() => undefined);
    }
    await db.delete(activityLog);
    await db.delete(environmentLeases);
    await db.delete(issues);
    await db.delete(costEvents);
    await db.delete(heartbeatRuns);
    await db.delete(agents);
    await db.delete(environments);
    await db.delete(executionWorkspaces);
    await db.delete(plugins);
    await db.delete(companySecretVersions);
    await db.delete(companySecrets);
    await db.delete(projects);
    await db.delete(companies);
  });

  afterAll(async () => {
    await stopDb?.();
  });

  async function seedEnvironment(input: {
    driver?: string;
    name?: string;
    status?: "active" | "disabled";
    config?: Record<string, unknown>;
  } = {}) {
    const companyId = randomUUID();
    const agentId = randomUUID();
    const environmentId = randomUUID();
    const runId = randomUUID();
    const driver = input.driver ?? "local";
    const environmentName = input.name ?? `${driver}-${environmentId.slice(0, 8)}`;
    let config = input.config ?? {};

    await db.insert(companies).values({
      id: companyId,
      name: "Acme",
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(agents).values({
      id: agentId,
      companyId,
      name: "CodexCoder",
      role: "engineer",
      status: "active",
      adapterType: "codex_local",
      adapterConfig: {},
      runtimeConfig: {},
      permissions: {},
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    if (typeof config.privateKey === "string" && config.privateKey.length > 0) {
      const secret = await secretService(db).create(companyId, {
        name: `environment-runtime-private-key-${randomUUID()}`,
        provider: "local_encrypted",
        value: config.privateKey,
      });
      await secretService(db).createBinding({
        companyId,
        secretId: secret.id,
        targetType: "environment",
        targetId: environmentId,
        configPath: "privateKeySecretRef",
      });
      config = {
        ...config,
        privateKey: null,
        privateKeySecretRef: {
          type: "secret_ref",
          secretId: secret.id,
          version: "latest",
        },
      };
    }
    const existingLocalEnvironment = driver === "local"
      ? await db
        .select()
        .from(environments)
        .where(eq(environments.driver, "local"))
        .then((rows) => rows[0] ?? null)
      : null;
    const environmentRecord = existingLocalEnvironment ?? {
      id: environmentId,
      name: environmentName,
      description: null,
      driver,
      status: input.status ?? "active",
      config,
      metadata: null,
      createdAt: new Date(),
      updatedAt: new Date(),
    };
    if (!existingLocalEnvironment) {
      await db.insert(environments).values({
        id: environmentRecord.id,
        name: environmentRecord.name,
        driver: environmentRecord.driver,
        status: environmentRecord.status,
        config: environmentRecord.config,
        createdAt: environmentRecord.createdAt,
        updatedAt: environmentRecord.updatedAt,
      });
    }
    await db.insert(heartbeatRuns).values({
      id: runId,
      companyId,
      agentId,
      invocationSource: "manual",
      status: "running",
      createdAt: new Date(),
      updatedAt: new Date(),
    });

    return {
      companyId,
      agentId,
      environment: {
        id: environmentRecord.id,
        companyId,
        name: environmentRecord.name,
        description: environmentRecord.description,
        driver: environmentRecord.driver,
        status: environmentRecord.status,
        config: environmentRecord.config,
        metadata: environmentRecord.metadata,
        createdAt: environmentRecord.createdAt,
        updatedAt: environmentRecord.updatedAt,
      } as const,
      runId,
    };
  }

  async function seedReusablePluginSandboxLease(adapterType: string | null = null) {
    const pluginId = randomUUID();
    const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
    const providerConfig = {
      provider: "fake-plugin",
      image: "fake:test",
      timeoutMs: 1234,
      reuseLease: true,
    };
    const environment = {
      ...baseEnvironment,
      name: "Reusable Plugin Sandbox",
      driver: "sandbox",
      config: providerConfig,
    };
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: providerConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "acme.reusable-sandbox-provider",
      packageName: "@acme/reusable-sandbox-provider",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.reusable-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Reusable Sandbox Provider",
        description: "Test provider with reusable lease support",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            supportsReusableLeases: true,
            configSchema: {
              type: "object",
              properties: {
                image: { type: "string" },
                timeoutMs: { type: "number" },
                reuseLease: { type: "boolean" },
              },
            },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);
    const executionWorkspaceId = randomUUID();
    const projectId = randomUUID();
    await db.insert(projects).values({
      id: projectId,
      companyId,
      name: `Workspace ${projectId.slice(0, 8)}`,
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(executionWorkspaces).values({
      id: executionWorkspaceId,
      companyId,
      projectId,
      mode: "shared_workspace",
      strategyType: "project_primary",
      name: "Reusable workspace",
      status: "active",
      providerType: "local_fs",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    const reusableLease = await environmentService(db).acquireLease({
      companyId,
      environmentId: environment.id,
      executionWorkspaceId,
      heartbeatRunId: runId,
      leasePolicy: "reuse_by_environment",
      provider: "fake-plugin",
      providerLeaseId: "reusable-plugin-lease",
      metadata: {
        agentId,
        driver: "sandbox",
        pluginId,
        pluginKey: "acme.reusable-sandbox-provider",
        sandboxProviderPlugin: true,
        provider: "fake-plugin",
        image: "fake:test",
        timeoutMs: 1234,
        reuseLease: true,
        reusableSandboxLease: {
          version: 1,
          companyId,
          environmentId: environment.id,
          executionWorkspaceId,
          agentId,
          adapterType,
          provider: "fake-plugin",
          runtimeFingerprint: reusableRuntimeFingerprint({
            provider: "fake-plugin",
            adapterType,
            config: providerConfig,
          }),
        },
      },
    });

    return { pluginId, companyId, agentId, environment, runId, executionWorkspaceId, reusableLease };
  }

  it("acquires and resumes a reusable lease after a task changes from per-turn to warm", async () => {
    const seeded = await seedReusablePluginSandboxLease("paperclip_runner");
    await environmentService(db).releaseLease(seeded.reusableLease.id, "expired");
    const environment = { ...seeded.environment, config: { ...seeded.environment.config, reuseLease: false } };
    await environmentService(db).update(environment.id, { config: environment.config });
    const issueId = randomUUID();
    await db.insert(issues).values({ id: issueId, companyId: seeded.companyId, title: "Lifecycle switch", status: "in_progress", assigneeAgentId: seeded.agentId });
    let acquisitions = 0;
    let warmProviderLeaseId = "";
    const call = vi.fn(async (_pluginId: string, method: string, input: any) => {
      if (method === "environmentAcquireLease") return {
        providerLeaseId: `lifecycle-${++acquisitions}`, metadata: { remoteCwd: "/workspace" },
      };
      if (method === "environmentResumeLease") return {
        providerLeaseId: warmProviderLeaseId, metadata: { remoteCwd: "/workspace" },
      };
      if (method === "environmentDestroyLease" || method === "environmentStopLease" || method === "environmentReleaseLease") return {
        providerLeaseId: input.providerLeaseId, state: method === "environmentDestroyLease" ? "destroyed" : "stopped",
      };
      throw new Error(`Unexpected lifecycle method: ${method}`);
    });
    const runtime = environmentRuntimeService(db, { pluginWorkerManager: {
      isRunning: () => true, call,
      getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentStopLease", "environmentDestroyLease"] }),
    } as unknown as PluginWorkerManager });
    const input = {
      companyId: seeded.companyId, agentId: seeded.agentId, issueId,
      adapterType: "paperclip_runner", persistedExecutionWorkspace: { id: seeded.executionWorkspaceId, mode: "shared_workspace" as const },
    };
    const cold = await runtime.acquireRunLease({ ...input, environment, heartbeatRunId: seeded.runId });
    expect(cold.lease.leasePolicy).toBe("ephemeral");
    await runtime.releaseRunLeases(seeded.runId, "released");
    const warmEnvironment = resolveRunnerEnvironmentForRun(environment, "paperclip_runner", { lifecycleMode: "warm" });
    const warmRunId = randomUUID();
    await db.insert(heartbeatRuns).values({ id: warmRunId, companyId: seeded.companyId, agentId: seeded.agentId, status: "running" });
    const warm = await runtime.acquireRunLease({ ...input, environment: warmEnvironment, heartbeatRunId: warmRunId });
    warmProviderLeaseId = warm.lease.providerLeaseId!;
    expect(warm.lease.leasePolicy).toBe("reuse_by_environment");
    expect(warm.lease.executionWorkspaceId).toBe(seeded.executionWorkspaceId);
    expect((await runtime.resolveCapabilities({ environment: warmEnvironment, lease: warm.lease })).reusableLeases).toBe(true);
    await runtime.releaseRunLeases(warmRunId, "released", undefined, "stop_and_retain");
    const nextRunId = randomUUID();
    await db.insert(heartbeatRuns).values({ id: nextRunId, companyId: seeded.companyId, agentId: seeded.agentId, status: "running" });
    const next = await runtime.acquireRunLease({ ...input, environment: warmEnvironment, heartbeatRunId: nextRunId });
    expect(next.lease.providerLeaseId).toBe(warmProviderLeaseId);
    expect(acquisitions).toBe(2);
    expect((await environmentService(db).getById(environment.id))?.config.reuseLease).toBe(false);
  });

  it.each(["stopped", "missing", "wrong-lease", "error"])(
    "startup cancellation is run-scoped and needs a matching receipt: %s",
    async (outcome) => {
      const { pluginId, companyId, runId, reusableLease, environment } = await seedReusablePluginSandboxLease();
      const other = await environmentService(db).acquireLease({
        companyId, environmentId: environment.id, heartbeatRunId: null,
        leasePolicy: "ephemeral", provider: "fake-plugin", providerLeaseId: "other-sandbox",
      });
      const call = vi.fn(async () => {
        if (outcome === "error") throw new Error("provider unavailable");
        if (outcome === "missing") return undefined;
        return { providerLeaseId: outcome === "wrong-lease" ? "other-sandbox" : reusableLease.providerLeaseId, state: "stopped" };
      });
      const workerManager = {
        isRunning: () => true, call,
        getWorker: () => ({ supportedMethods: ["environmentStopLease"] }),
      } as unknown as PluginWorkerManager;
      const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
      await runtimeWithPlugin.releaseRunLeases(runId, "released", undefined, "stop_and_retain", true);
      expect(call).toHaveBeenCalledWith(pluginId, "environmentStopLease", expect.objectContaining({
        companyId, providerLeaseId: reusableLease.providerLeaseId, cancelActiveWork: true,
      }), expect.any(Number));
      expect(call).toHaveBeenCalledOnce();
      expect(await remoteExecutionHasStopped(db, companyId, runId)).toBe(outcome === "stopped");
      await expect(environmentService(db).getLeaseById(other.id)).resolves.toMatchObject({ status: "active" });
    },
  );

  it.each(["release_only", "stopped", "stop_failed", "unconfirmed", "restart", "competing_owner", "missing_pin", "late_receipt"])(
    "never dispatches destructive release for an untagged stop-and-retain request: %s", async outcome => {
      const seeded = await seedReusablePluginSandboxLease("paperclip_runner");
      const lease = seeded.reusableLease;
      await db.update(environmentLeases).set({ leasePolicy: "ephemeral", metadata: {
        ...lease.metadata, reuseLease: false, ...(outcome === "missing_pin" ? { pluginId: undefined } : {}),
      } }).where(eq(environmentLeases.id, lease.id));
      await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId));
      let supported = !["release_only", "restart"].includes(outcome);
      let calls = 0;
      const call = vi.fn(async (_id: string, method: string) => {
        if (method !== "environmentStopLease") return { providerLeaseId: lease.providerLeaseId, state: "destroyed" };
        if (++calls === 1 && outcome === "stop_failed") throw new Error("injected stop failure");
        if (calls === 1 && outcome === "unconfirmed") return undefined;
        if (outcome === "late_receipt") await db.update(environmentLeases).set({
          status: "active", heartbeatRunId: null, cleanupStatus: null, metadata: { newOwner: true },
        }).where(eq(environmentLeases.id, lease.id));
        return { providerLeaseId: lease.providerLeaseId, state: "stopped" };
      });
      const runtime = () => environmentRuntimeService(db, { pluginWorkerManager: {
        isRunning: () => true, call,
        getWorker: () => ({ supportedMethods: ["environmentReleaseLease", "environmentDestroyLease", ...(supported ? ["environmentStopLease"] : [])] }),
      } as unknown as PluginWorkerManager });
      if (outcome === "competing_owner") await environmentService(db).acquireLease({
        companyId: seeded.companyId, environmentId: seeded.environment.id, heartbeatRunId: null,
        leasePolicy: "ephemeral", provider: lease.provider, providerLeaseId: lease.providerLeaseId,
      });
      await runtime().releaseRunLeases(seeded.runId, "failed", undefined, "stop_and_retain");
      expect(call.mock.calls.every(entry => entry[1] === "environmentStopLease")).toBe(true);
      if (outcome === "late_receipt") {
        expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "active", heartbeatRunId: null, metadata: { newOwner: true } });
        return;
      }
      if (outcome === "missing_pin") {
        expect(call).not.toHaveBeenCalled();
        await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 });
        expect(call).not.toHaveBeenCalled();
        expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "pending_cleanup" });
        return;
      }
      if (["release_only", "restart", "competing_owner"].includes(outcome)) {
        expect(call).not.toHaveBeenCalled();
        expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "pending_cleanup", metadata: {
          sandboxStopAndRetain: { pluginId: seeded.pluginId, runId: seeded.runId, providerLeaseId: lease.providerLeaseId },
        } });
        await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 });
        expect(call).not.toHaveBeenCalled();
        if (outcome === "competing_owner") return;
        supported = true;
      }
      if (["release_only", "stop_failed", "unconfirmed", "restart"].includes(outcome)) {
        await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 });
      }
      expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "released", cleanupStatus: "success", failureReason: null,
        metadata: { remoteExecutionTermination: { state: "stopped", providerLeaseId: lease.providerLeaseId },
          sandboxStopAndRetainReceipt: { method: "environmentStopLease", runId: seeded.runId, pluginId: seeded.pluginId } } });
      expect(call.mock.calls.every(entry => entry[1] === "environmentStopLease")).toBe(true);
    },
  );

  it("reports an unwired manager separately from a stopped sandbox worker", async () => {
    const { companyId, environment, runId } = await seedReusablePluginSandboxLease();
    const input = {
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
    };
    const runtimeWithoutManager = environmentRuntimeService(db);
    await expect(runtimeWithoutManager.acquireRunLease(input))
      .rejects.toThrow("sandbox plugin workers are unavailable in this server process");

    const offlineManager = { isRunning: () => false, call: vi.fn() } as unknown as PluginWorkerManager;
    const runtimeWithStoppedWorker = environmentRuntimeService(db, {
      pluginWorkerManager: offlineManager,
      pluginWorkerReadyTimeoutMs: 0,
    });
    await expect(runtimeWithStoppedWorker.acquireRunLease(input))
      .rejects.toThrow("its worker is not running");
    expect(offlineManager.call).not.toHaveBeenCalled();
  });

  it("retains a successful reusable sandbox lease without stopping the provider resource", async () => {
    const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease();
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        throw new Error(`Unexpected plugin method while retaining warm lease: ${method}`);
      }),
      getWorker: vi.fn(() => ({
        supportedMethods: [
          "environmentResumeLease",
          "environmentReleaseLease",
          "environmentDestroyLease",
        ],
      })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const released = await runtimeWithPlugin.releaseRunLeases(
      runId,
      "released",
      undefined,
      "keep_running",
    );

    expect(released).toHaveLength(1);
    expect(released[0]?.lease).toMatchObject({
      id: reusableLease.id,
      status: "retained",
      cleanupStatus: "success",
    });
    await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
      status: "retained",
      cleanupStatus: "success",
    });
    expect(workerManager.call).not.toHaveBeenCalled();
  });

  it("stops a reusable sandbox and keeps its lease resumable", async () => {
    const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease();
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentStopLease") return { providerLeaseId: reusableLease.providerLeaseId, state: "stopped" };
        throw new Error(`Unexpected plugin method while stopping lease: ${method}`);
      }),
      getWorker: vi.fn(() => ({
        supportedMethods: [
          "environmentResumeLease",
          "environmentReleaseLease",
          "environmentStopLease",
          "environmentDestroyLease",
        ],
      })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const released = await runtimeWithPlugin.releaseRunLeases(
      runId,
      "failed",
      undefined,
      "stop_and_retain",
    );

    expect(released[0]?.lease).toMatchObject({
      id: reusableLease.id,
      status: "released",
    });
    expect(workerManager.call).toHaveBeenCalledWith(
      pluginId,
      "environmentStopLease",
      expect.objectContaining({ providerLeaseId: reusableLease.providerLeaseId }),
      expect.any(Number),
    );
  });

  it.each(["same task", "different task", "different agent", "no task"])(
    "scopes projectless reusable leases to their task and agent: %s", async (scenario) => {
    const seeded = await seedReusablePluginSandboxLease();
    await environmentService(db).releaseLease(seeded.reusableLease.id, "expired");
    const issueId = randomUUID();
    const otherIssueId = randomUUID();
    await db.insert(issues).values([issueId, otherIssueId].map(id => ({
      id, companyId: seeded.companyId, title: "Projectless chat", status: "in_progress",
      assigneeAgentId: seeded.agentId,
    })));
    const nextAgentId = scenario === "different agent" ? randomUUID() : seeded.agentId;
    if (nextAgentId !== seeded.agentId) await db.insert(agents).values({
      id: nextAgentId, companyId: seeded.companyId, name: "Other agent", adapterType: "paperclip_runner",
    });
    let acquisitions = 0;
    const call = vi.fn(async (_pluginId: string, method: string) => {
      if (method === "environmentAcquireLease") return {
        providerLeaseId: `projectless-${++acquisitions}`, metadata: { remoteCwd: "/workspace" },
      };
      if (method === "environmentResumeLease") return {
        providerLeaseId: "projectless-1", metadata: { remoteCwd: "/workspace" },
      };
      if (method === "environmentStopLease") return { providerLeaseId: "projectless-1", state: "stopped" };
      throw new Error(`Unexpected projectless lease method: ${method}`);
    });
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: {
      isRunning: () => true, call,
      getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentStopLease", "environmentDestroyLease"] }),
    } as unknown as PluginWorkerManager });
    const first = await runtimeWithPlugin.acquireRunLease({
      companyId: seeded.companyId, environment: seeded.environment, issueId,
      agentId: seeded.agentId, heartbeatRunId: seeded.runId, persistedExecutionWorkspace: null,
    });
    expect(first.lease.metadata?.reusableSandboxLease).toMatchObject({
      projectlessIssueId: issueId, executionWorkspaceId: null, agentId: seeded.agentId,
    });
    await runtimeWithPlugin.releaseRunLeases(seeded.runId, "released", undefined, "stop_and_retain");
    const nextRunId = randomUUID();
    await db.insert(heartbeatRuns).values({
      id: nextRunId, companyId: seeded.companyId, agentId: nextAgentId, status: "running",
    });
    if (scenario === "different task" || scenario === "different agent") {
      await expect(environmentService(db).acquireLease({
        companyId: seeded.companyId, environmentId: seeded.environment.id,
        issueId: scenario === "different task" ? otherIssueId : issueId,
        heartbeatRunId: nextRunId, executionWorkspaceId: null,
        provider: "fake-plugin", providerLeaseId: first.lease.providerLeaseId,
        leasePolicy: "reuse_by_environment", replacesReusableLeaseId: first.lease.id,
        metadata: { agentId: nextAgentId },
      })).rejects.toMatchObject({ status: 409 });
    }
    const next = await runtimeWithPlugin.acquireRunLease({
      companyId: seeded.companyId, environment: seeded.environment,
      issueId: scenario === "no task" ? null : scenario === "different task" ? otherIssueId : issueId,
      agentId: nextAgentId, heartbeatRunId: nextRunId, persistedExecutionWorkspace: null,
    });
    const shouldResume = scenario === "same task";
    expect(next.lease.providerLeaseId).toBe(shouldResume ? "projectless-1" : "projectless-2");
    expect(acquisitions).toBe(shouldResume ? 1 : 2);
    expect(call.mock.calls.filter((entry) => entry[1] === "environmentResumeLease")).toHaveLength(shouldResume ? 1 : 0);
  });

  it.each(["pending", "inline", "foreign scope", "foreign environment", "foreign run", "malformed", "deleted environment", "forged error", "invalid diagnostic", "old worker"])("retains uncertain plugin creation for durable cleanup: %s", async (mode) => {
    const seeded = await seedReusablePluginSandboxLease();
    await environmentService(db).releaseLease(seeded.reusableLease.id, "expired");
    const attemptId = randomUUID();
    const providerLeaseId = `paperclip-create-${attemptId}`;
    const cleanup = {
      providerLeaseId, attemptId, companyId: mode === "foreign scope" ? randomUUID() : seeded.companyId,
      environmentId: mode === "foreign environment" ? randomUUID() : seeded.environment.id,
      runId: mode === "foreign run" ? randomUUID() : seeded.runId,
      accountFingerprint: mode === "malformed" ? "invalid" : "a".repeat(64), labels: { "paperclip-provider": "fake-plugin" },
    };
    const acquisitionDiagnostic = { phase: "shell", elapsedMs: 300_000, budgetMs: 300_000 };
    const data = { schema: "paperclip/environment-creation-cleanup/v1", cleanup,
      ...(mode === "old worker" ? {} : { acquisitionDiagnostic: mode === "invalid diagnostic"
        ? { ...acquisitionDiagnostic, phase: "private-command" } : acquisitionDiagnostic }),
    };
    const failure = mode === "forged error"
      ? Object.assign(new Error("Provider creation cleanup required"), { data })
      : new JsonRpcCallError({ code: -32002, message: "Provider creation cleanup required", data });
    let available = mode === "inline";
    const call = vi.fn(async (_id: string, method: string, params: Record<string, unknown>) => {
      if (method === "environmentAcquireLease") throw failure;
      if (method === "environmentDestroyLease") {
        const pending = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, providerLeaseId));
        expect(pending).toHaveLength(1);
        expect(pending[0].status).toBe("pending_cleanup");
        expect(params).toMatchObject({ providerLeaseId, companyId: seeded.companyId, environmentId: seeded.environment.id, leaseMetadata: { failedCreateCleanup: cleanup } });
        if (!available) throw new Error("provider not visible yet");
        return { providerLeaseId, state: "destroyed" };
      }
      throw new Error(`Unexpected method ${method}`);
    });
    const worker = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }), call } as unknown as PluginWorkerManager;
    const candidate = environmentRuntimeService(db, { pluginWorkerManager: worker });
    const acquisition = candidate.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment,
      issueId: null, heartbeatRunId: seeded.runId, persistedExecutionWorkspace: null });
    const caught = await acquisition.catch(error => error);
    const execution = collectRunFailureDiagnostics({ id: seeded.runId, companyId: seeded.companyId,
      errorCode: "setup_failed", resultJson: null,
    } as typeof heartbeatRuns.$inferSelect, { error: caught, phase: "setup" }).execution;
    if (["pending", "inline", "deleted environment"].includes(mode)) {
      expect(execution).toMatchObject({ environmentAcquisitionPhase: "shell",
        environmentAcquisitionElapsedMs: 300_000, environmentAcquisitionBudgetMs: 300_000 });
    } else expect(execution).not.toHaveProperty("environmentAcquisitionPhase");
    if (mode === "inline") {
      await expect(acquisition).rejects.toMatchObject({
        message: "Sandbox creation failed; allocated sandbox cleanup was confirmed.", cause: failure,
      });
    } else {
      await expect(acquisition).rejects.toBe(failure);
    }
    const rows = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, providerLeaseId));
    if (["foreign scope", "foreign environment", "foreign run", "malformed"].includes(mode)) {
      expect(rows).toHaveLength(0);
      expect(call.mock.calls.filter((c) => c[1] === "environmentDestroyLease")).toHaveLength(0);
      return;
    }
    expect(rows).toHaveLength(1);
    expect(rows[0]).toMatchObject({ status: mode === "inline" ? "expired" : "pending_cleanup",
      cleanupStatus: mode === "inline" ? "success" : "failed", companyId: seeded.companyId,
      metadata: { pluginId: seeded.pluginId, sandboxProviderPlugin: true, failedCreateCleanup: cleanup } });
    if (mode === "pending" || mode === "deleted environment") {
      available = true;
      if (mode === "deleted environment") {
        await db.update(environmentLeases).set({ environmentId: null }).where(eq(environmentLeases.id, rows[0].id));
      }
      const restarted = environmentRuntimeService(db, { pluginWorkerManager: worker });
      const persisted = await environmentService(db).getLeaseById(rows[0].id);
      await expect(restarted.retryPendingSandboxTeardown({ environment: null, lease: persisted! })).resolves.toEqual({ providerLeaseId, state: "destroyed" });
    }
  });

  it.each(["deleted receipt lost", "observation response lost", "journal write failed", "foreign observation"])(
    "journals observed creation identity before deletion and recovers: %s", async (fault) => {
      const seeded = await seedReusablePluginSandboxLease();
      await environmentService(db).releaseLease(seeded.reusableLease.id, "expired");
      const cleanup = {
        providerLeaseId: `paperclip-create-${randomUUID()}`, attemptId: randomUUID(),
        companyId: seeded.companyId, environmentId: seeded.environment.id, runId: seeded.runId,
        accountFingerprint: "a".repeat(64), labels: { "paperclip-provider": "fake-plugin" },
      };
      const observed = { ...cleanup, observedProviderLeaseId: "verified-provider-id" };
      const failure = Object.assign(new Error("Creation uncertain"), {
        data: { schema: "paperclip/environment-creation-cleanup/v1", cleanup },
      });
      let injected = false;
      let deleted = false;
      const call = vi.fn(async (_id: string, method: string, params: Record<string, unknown>) => {
        if (method === "environmentAcquireLease") throw failure;
        if (method !== "environmentDestroyLease") throw new Error(`Unexpected ${method}`);
        const evidence = (params.leaseMetadata as { failedCreateCleanup: typeof observed }).failedCreateCleanup;
        if (!evidence.observedProviderLeaseId) {
          expect(deleted).toBe(false);
          if (!injected && fault === "observation response lost") {
            injected = true;
            throw new Error("Lost observation response");
          }
          if (!injected && fault === "foreign observation") {
            injected = true;
            throw Object.assign(new Error("Foreign observation"), {
              data: { schema: "paperclip/environment-creation-cleanup/v1", cleanup: { ...observed, companyId: "another-company" } },
            });
          }
          if (!injected && fault === "journal write failed") {
            injected = true;
            vi.spyOn(db, "update").mockImplementationOnce(() => { throw new Error("Database unavailable"); });
          }
          throw Object.assign(new Error("Persist observed sandbox identity before cleanup"), {
            data: { schema: "paperclip/environment-creation-cleanup/v1", cleanup: observed },
          });
        }
        const rows = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, cleanup.providerLeaseId));
        expect(rows).toHaveLength(1);
        expect(rows[0].metadata).toMatchObject({ failedCreateCleanup: observed });
        if (!deleted) {
          deleted = true;
          if (fault === "deleted receipt lost") throw new Error("Deletion succeeded but reply lost");
        }
        return { providerLeaseId: cleanup.providerLeaseId, state: "destroyed" };
      });
      const worker = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }), call } as unknown as PluginWorkerManager;
      const runtime = environmentRuntimeService(db, { pluginWorkerManager: worker });
      await expect(runtime.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment,
        issueId: null, heartbeatRunId: seeded.runId, persistedExecutionWorkspace: null })).rejects.toBe(failure);
      const rows = await db.select().from(environmentLeases).where(eq(environmentLeases.providerLeaseId, cleanup.providerLeaseId));
      expect(rows).toHaveLength(1);
      expect(rows[0].status).toBe("pending_cleanup");
      expect(deleted).toBe(fault === "deleted receipt lost");
      const restarted = environmentRuntimeService(db, { pluginWorkerManager: worker });
      await expect(restarted.retryPendingSandboxTeardown({ environment: null, lease: rows[0]! })).resolves.toEqual({ providerLeaseId: cleanup.providerLeaseId, state: "destroyed" });
      expect(deleted).toBe(true);
    },
  );

  it("resumes the same provider lease on the next per-turn run", async () => {
    const seeded = await seedReusablePluginSandboxLease();
    const workerManager = {
      isRunning: vi.fn((id: string) => id === seeded.pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "sandbox-exact-resume",
            metadata: {
              provider: "fake-plugin",
              image: "fake:test",
              timeoutMs: 1234,
              reuseLease: true,
              remoteCwd: "/workspace",
            },
          };
        }
        if (method === "environmentStopLease") return { providerLeaseId: "sandbox-exact-resume", state: "stopped" };
        if (method === "environmentResumeLease") {
          return {
            providerLeaseId: "sandbox-exact-resume",
            metadata: {
              provider: "fake-plugin",
              image: "fake:test",
              timeoutMs: 1234,
              reuseLease: true,
              remoteCwd: "/workspace",
            },
          };
        }
        throw new Error(`Unexpected plugin method during exact resume: ${method}`);
      }),
      getWorker: vi.fn(() => ({
        supportedMethods: [
          "environmentResumeLease",
          "environmentReleaseLease",
          "environmentStopLease",
          "environmentDestroyLease",
        ],
      })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
    await environmentService(db).releaseLease(seeded.reusableLease.id, "expired");
    const first = await runtimeWithPlugin.acquireRunLease({
      companyId: seeded.companyId,
      environment: seeded.environment,
      issueId: null,
      agentId: seeded.agentId,
      heartbeatRunId: seeded.runId,
      persistedExecutionWorkspace: {
        id: seeded.executionWorkspaceId,
        mode: "shared_workspace",
      },
    });
    const workspaceSyncStamp = {
      schema: "paperclip.native-workspace-stamp/v1",
      workspaceId: seeded.executionWorkspaceId,
      providerLeaseId: "sandbox-exact-resume",
      remoteCwd: "/workspace",
      hostSha256: "a".repeat(64),
      finalizedRunId: seeded.runId,
    };
    await environmentService(db).updateLeaseMetadata(first.lease.id, {
      ...(first.lease.metadata ?? {}),
      nativeWorkspaceSync: workspaceSyncStamp,
    });
    await runtimeWithPlugin.releaseRunLeases(
      seeded.runId,
      "released",
      undefined,
      "stop_and_retain",
    );

    const nextRunId = randomUUID();
    await db.insert(heartbeatRuns).values({
      id: nextRunId,
      companyId: seeded.companyId,
      agentId: seeded.agentId,
      invocationSource: "manual",
      status: "running",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    const acquired = await runtimeWithPlugin.acquireRunLease({
      companyId: seeded.companyId,
      environment: seeded.environment,
      issueId: null,
      agentId: seeded.agentId,
      heartbeatRunId: nextRunId,
      persistedExecutionWorkspace: {
        id: seeded.executionWorkspaceId,
        mode: "shared_workspace",
      },
    });

    expect(first.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "created" });
    expect(acquired.lease.providerLeaseId).toBe("sandbox-exact-resume");
    expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({
      outcome: "resumed",
    });
    expect(acquired.lease.metadata?.nativeWorkspaceSync).toEqual(
      workspaceSyncStamp,
    );
    await expect(
      environmentService(db).getLeaseById(first.lease.id),
    ).resolves.toMatchObject({
      status: "expired",
      cleanupStatus: "success",
    });
    expect(
      workerManager.call.mock.calls.filter(
        (call) => call[1] === "environmentAcquireLease",
      ),
    ).toHaveLength(1);
  });

  it("reacquires one provider lease row idempotently during same-run recovery", async () => {
    const seeded = await seedReusablePluginSandboxLease();
    const workerManager = {
      isRunning: vi.fn((id: string) => id === seeded.pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentResumeLease") {
          return {
            providerLeaseId: seeded.reusableLease.providerLeaseId,
            metadata: {
              provider: "fake-plugin",
              image: "fake:test",
              timeoutMs: 1234,
              reuseLease: true,
              remoteCwd: "/workspace",
            },
          };
        }
        throw new Error(
          `Unexpected plugin method during same-run reacquisition: ${method}`,
        );
      }),
      getWorker: vi.fn(() => ({
        supportedMethods: [
          "environmentResumeLease",
          "environmentReleaseLease",
          "environmentDestroyLease",
        ],
      })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, {
      pluginWorkerManager: workerManager,
    });

    const reacquired = await runtimeWithPlugin.acquireRunLease({
      companyId: seeded.companyId,
      environment: seeded.environment,
      issueId: null,
      agentId: seeded.agentId,
      heartbeatRunId: seeded.runId,
      persistedExecutionWorkspace: {
        id: seeded.executionWorkspaceId,
        mode: "shared_workspace",
      },
    });

    expect(reacquired.lease).toMatchObject({
      id: seeded.reusableLease.id,
      heartbeatRunId: seeded.runId,
      providerLeaseId: seeded.reusableLease.providerLeaseId,
      status: "active",
      cleanupStatus: null,
    });
    const rows = await environmentService(db).listLeases(seeded.environment.id);
    expect(
      rows.filter(
        (lease) =>
          lease.heartbeatRunId === seeded.runId &&
          lease.providerLeaseId === seeded.reusableLease.providerLeaseId,
      ),
    ).toHaveLength(1);
    expect(workerManager.call).toHaveBeenCalledWith(
      seeded.pluginId,
      "environmentResumeLease",
      expect.objectContaining({
        providerLeaseId: seeded.reusableLease.providerLeaseId,
      }),
      expect.any(Number),
    );
  });

  it("destroys a disposable paperclip_runner sandbox after the turn", async () => {
    const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease();
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentDestroyLease") return undefined;
        throw new Error(`Unexpected plugin method while destroying lease: ${method}`);
      }),
      getWorker: vi.fn(() => ({
        supportedMethods: [
          "environmentResumeLease",
          "environmentReleaseLease",
          "environmentDestroyLease",
        ],
      })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const released = await runtimeWithPlugin.releaseRunLeases(
      runId,
      "released",
      undefined,
      "destroy",
    );

    expect(released[0]?.lease).toMatchObject({
      id: reusableLease.id,
      status: "expired",
    });
    expect(workerManager.call).toHaveBeenCalledWith(
      pluginId,
      "environmentDestroyLease",
      expect.objectContaining({ providerLeaseId: reusableLease.providerLeaseId }),
      expect.any(Number),
    );
  });

  it("does not destroy native sandbox state when its verified backup is missing", async () => {
    const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease();
    await environmentService(db).updateLeaseMetadata(reusableLease.id, {
      ...(reusableLease.metadata ?? {}),
      reusableSandboxLease: {
        ...((reusableLease.metadata?.reusableSandboxLease as Record<
          string,
          unknown
        >) ?? {}),
        adapterType: "paperclip_runner",
      },
      sandboxLeaseAcquisition: { outcome: "created" },
    });
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        throw new Error(`Provider teardown must not run without a backup: ${method}`);
      }),
      getWorker: vi.fn(() => ({
        supportedMethods: [
          "environmentResumeLease",
          "environmentReleaseLease",
          "environmentDestroyLease",
        ],
      })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
    const errors: unknown[] = [];

    const released = await runtimeWithPlugin.releaseRunLeases(
      runId,
      "released",
      (_leaseId, error) => errors.push(error),
      "destroy",
    );

    expect(released).toEqual([]);
    expect(errors).toHaveLength(1);
    expect(String(errors[0])).toContain("runner_harness_backup_unavailable");
    expect(workerManager.call).not.toHaveBeenCalled();
    await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
      status: "active",
    });
  });

  it("acquires and releases a local run lease through the runtime seam", async () => {
    const { companyId, environment, runId } = await seedEnvironment();

    const acquired = await runtime.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
    });

    expect(acquired.lease.status).toBe("active");
    expect(acquired.lease.metadata).toMatchObject({
      driver: "local",
      executionWorkspaceMode: null,
    });
    expect(acquired.leaseContext).toEqual({
      executionWorkspaceId: null,
      executionWorkspaceMode: null,
    });

    const released = await runtime.releaseRunLeases(runId);

    expect(released).toHaveLength(1);
    expect(released[0]?.environment.driver).toBe("local");
    expect(released[0]?.lease.status).toBe("released");

    const rows = await db
      .select()
      .from(environmentLeases)
      .where(eq(environmentLeases.id, acquired.lease.id));
    expect(rows[0]?.status).toBe("released");
  });

  it("allows projectless runs through the runtime seam", async () => {
    const { companyId, environment, runId } = await seedEnvironment();

    const acquired = await runtime.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
    });

    expect(acquired.lease.executionWorkspaceId).toBeNull();
    expect(acquired.leaseContext.executionWorkspaceId).toBeNull();
    expect(acquired.leaseContext.executionWorkspaceMode).toBeNull();
  });

  it("rejects truly unsupported drivers before acquiring a lease", async () => {
    const { companyId, agentId, environment, runId } = await seedEnvironment({
      driver: "ssh",
      name: "Fixture SSH",
      config: {
        host: "ssh.example.test",
        port: 22,
        username: "ssh-user",
        remoteWorkspacePath: "/srv/paperclip/workspace",
        privateKey: null,
        knownHosts: null,
        strictHostKeyChecking: true,
      },
    });
    const runtimeWithoutSsh = environmentRuntimeService(db, {
      drivers: [
        {
          driver: "local",
          acquireRunLease: async () => {
            throw new Error("should not acquire");
          },
          releaseRunLease: async () => null,
        },
      ],
    });

    await expect(
      runtimeWithoutSsh.acquireRunLease({
        companyId,
        environment,
        issueId: null,
        heartbeatRunId: runId,
        persistedExecutionWorkspace: null,
      }),
    ).rejects.toThrow('Environment driver "ssh" is not registered in the environment runtime yet.');

    const rows = await db.select().from(environmentLeases);
    expect(rows).toHaveLength(0);
  });

  it("acquires and releases an SSH run lease through the runtime seam", async () => {
    if (!sshFixtureSupport.supported) {
      console.warn(
        `Skipping SSH runtime fixture test: ${sshFixtureSupport.reason ?? "unsupported environment"}`,
      );
      return;
    }

    const fixtureRoot = await mkdtemp(path.join(os.tmpdir(), "paperclip-environment-runtime-ssh-"));
    fixtureRoots.push(fixtureRoot);
    const statePath = path.join(fixtureRoot, "state.json");
    const fixture = await startSshEnvLabFixture({ statePath });
    const sshConfig = await buildSshEnvLabFixtureConfig(fixture);
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "ssh",
      name: "Fixture SSH",
      config: sshConfig,
    });
    try {
      const acquired = await runtime.acquireRunLease({
        companyId,
        environment,
        issueId: null,
        heartbeatRunId: runId,
        persistedExecutionWorkspace: null,
      });

      expect(acquired.lease.status).toBe("active");
      expect(acquired.lease.providerLeaseId).toContain(`ssh://${sshConfig.username}@${sshConfig.host}:${sshConfig.port}`);
      expect(acquired.lease.metadata).toMatchObject({
        driver: "ssh",
        host: sshConfig.host,
        port: sshConfig.port,
        username: sshConfig.username,
        remoteWorkspacePath: sshConfig.remoteWorkspacePath,
        remoteCwd: sshConfig.remoteWorkspacePath,
      });

      const released = await runtime.releaseRunLeases(runId);

      expect(released).toHaveLength(1);
      expect(released[0]?.environment.driver).toBe("ssh");
      expect(released[0]?.lease.status).toBe("released");
    } finally {
    }
  });

  it("acquires and releases a fake sandbox run lease through the runtime seam", async () => {
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Fake Sandbox",
      config: {
        provider: "fake",
        image: "ubuntu:24.04",
        reuseLease: true,
      },
    });

    const acquired = await runtime.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
    });

    expect(acquired.lease.status).toBe("active");
    expect(acquired.lease.providerLeaseId).toMatch(new RegExp(`^sandbox://fake/${runId}/[0-9a-f-]{36}$`));
    expect(acquired.lease.leasePolicy).toBe("ephemeral");
    expect(acquired.lease.metadata).toMatchObject({
      driver: "sandbox",
      provider: "fake",
      image: "ubuntu:24.04",
      reuseLease: true,
    });

    const released = await runtime.releaseRunLeases(runId);

    expect(released).toHaveLength(1);
    expect(released[0]?.environment.driver).toBe("sandbox");
    expect(released[0]?.lease.status).toBe("released");
  });

  it.each([false, true])("confirms a built-in fake stop without release or destroy fallback: cancel=%s", async cancel => {
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
    });
    const acquired = await runtime.acquireRunLease({ companyId, environment, issueId: null,
      heartbeatRunId: runId, persistedExecutionWorkspace: null });
    const provider = sandboxProviderRuntime.requireSandboxProvider("fake");
    const release = vi.spyOn(provider, "releaseLease"), destroy = vi.spyOn(provider, "destroyLease");
    const released = await runtime.releaseRunLeases(runId, "released", undefined, "stop_and_retain", cancel);
    expect(released).toHaveLength(1);
    expect(released[0]?.lease).toMatchObject({ status: "released", cleanupStatus: "success", failureReason: null,
      metadata: { remoteExecutionTermination: { state: "stopped", providerLeaseId: acquired.lease.providerLeaseId },
        sandboxStopAndRetainReceipt: { method: "builtin.stopLease", builtinProvider: "fake", runId } } });
    expect(await remoteExecutionHasStopped(db, companyId, runId)).toBe(true);
    expect(release).not.toHaveBeenCalled();
    expect(destroy).not.toHaveBeenCalled();
  });

  it.each(["stop_failed", "wrong_receipt", "missing_capability", "changed_binding"])(
    "preserves a built-in stop through restart without destructive fallback: %s", async outcome => {
      const { companyId, environment, runId } = await seedEnvironment({
        driver: "sandbox", config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
      });
      const acquired = await runtime.acquireRunLease({ companyId, environment, issueId: null,
        heartbeatRunId: runId, persistedExecutionWorkspace: null });
      const provider = sandboxProviderRuntime.requireSandboxProvider("fake"), originalStop = provider.stopLease!;
      const release = vi.spyOn(provider, "releaseLease"), destroy = vi.spyOn(provider, "destroyLease");
      try {
        provider.stopLease = outcome === "missing_capability" ? undefined : vi.fn(async () => {
          if (outcome === "wrong_receipt") return { providerLeaseId: "sandbox://fake/other", state: "stopped" as const };
          throw new Error("injected built-in stop failure");
        });
        await runtime.releaseRunLeases(runId, "released", undefined, "stop_and_retain", true);
        expect(await environmentService(db).getLeaseById(acquired.lease.id)).toMatchObject({ status: "pending_cleanup" });
        expect(await remoteExecutionHasStopped(db, companyId, runId)).toBe(false);
        if (outcome === "missing_capability") {
          await heartbeatService(db, { environmentRuntime: environmentRuntimeService(db) }).sweepPendingCleanupLeases({ backoffMs: 0 });
          expect(await environmentService(db).getLeaseById(acquired.lease.id)).toMatchObject({ status: "pending_cleanup" });
        }
        const stop = vi.fn(originalStop.bind(provider));
        provider.stopLease = stop;
        if (outcome === "changed_binding") await db.update(environmentLeases).set({ providerLeaseId: "sandbox://fake/replacement" })
          .where(eq(environmentLeases.id, acquired.lease.id));
        await heartbeatService(db, { environmentRuntime: environmentRuntimeService(db) }).sweepPendingCleanupLeases({ backoffMs: 0 });
        if (outcome === "changed_binding") {
          expect(stop).not.toHaveBeenCalled();
          expect(await environmentService(db).getLeaseById(acquired.lease.id)).toMatchObject({ status: "pending_cleanup" });
        } else {
          expect(stop).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ providerLeaseId: acquired.lease.providerLeaseId }));
          expect(await environmentService(db).getLeaseById(acquired.lease.id)).toMatchObject({ status: "released", cleanupStatus: "success" });
          expect(await remoteExecutionHasStopped(db, companyId, runId)).toBe(true);
        }
        expect(release).not.toHaveBeenCalled();
        expect(destroy).not.toHaveBeenCalled();
      } finally { provider.stopLease = originalStop; }
    },
  );

  it("releases the remote sandbox when the lease insert rejects a foreign-company binding", async () => {
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Foreign-bound Fake Sandbox",
      config: {
        provider: "fake",
        image: "ubuntu:24.04",
        reuseLease: false,
      },
    });

    // A managed reconciliation binds the environment to another company after the
    // route guard read an empty binding list. The lease insert then rejects the
    // foreign-company binding with the 403 `environment_company_mismatch`.
    const otherCompanyId = randomUUID();
    await db.insert(companies).values({
      id: otherCompanyId,
      name: "Other Co",
      issuePrefix: "OTA",
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(builtInManagedResources).values({
      companyId: otherCompanyId,
      bundleKey: "managed-environment",
      resourceKind: "environment",
      resourceKey: "managed-sandbox",
      resourceId: environment.id,
      stockVersion: "1",
      stockHash: "hash",
    });

    const destroySpy = vi.spyOn(sandboxProviderRuntime, "destroySandboxProviderLease");
    try {
      await expect(
        runtime.acquireRunLease({
          companyId,
          environment,
          issueId: null,
          heartbeatRunId: runId,
          persistedExecutionWorkspace: null,
          assertCompanyBinding: true,
        }),
      ).rejects.toMatchObject({
        status: 403,
        details: { code: "environment_company_mismatch" },
      });

      // The acquire records the durable pending-cleanup row before the teardown,
      // so the row lands while the database is proven reachable. The successful
      // teardown then releases the row to the terminal `expired` state, so no
      // active or pending_cleanup row remains for the orphan.
      const leaseRows = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.environmentId, environment.id));
      expect(leaseRows).toHaveLength(1);
      expect(leaseRows[0]?.status).toBe("expired");
      expect(leaseRows[0]?.cleanupStatus).toBe("success");

      // The acquire already provisioned the remote sandbox, so it releases the
      // sandbox on the rejection. Without this teardown the rejected insert leaks
      // a live sandbox that no lease row tracks. The teardown carries the
      // provisioned provider lease id.
      expect(destroySpy).toHaveBeenCalledTimes(1);
      expect(destroySpy.mock.calls[0]?.[0]?.providerLeaseId).toMatch(
        new RegExp(`^sandbox://fake/${runId}/[0-9a-f-]{36}$`),
      );
    } finally {
      destroySpy.mockRestore();
    }
  });

  it("destroys the remote plugin sandbox when the lease insert rejects a foreign-company binding", async () => {
    // The Claude login runs on a plugin-backed sandbox provider (Daytona), so this
    // path is the one the login uses. It must also release the remote sandbox when
    // the conditional lease insert rejects a foreign-company binding.
    const pluginId = randomUUID();
    const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
    const pluginConfig = { provider: "fake-plugin", image: "fake:test", reuseLease: false };
    const environment = {
      ...baseEnvironment,
      name: "Foreign-bound Plugin Sandbox",
      driver: "sandbox",
      config: pluginConfig,
    };
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: pluginConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "paperclip.fake-plugin-sandbox-provider",
      packageName: "@paperclipai/plugin-fake-sandbox",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "paperclip.fake-plugin-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Fake Plugin Sandbox Provider",
        description: "Test fake plugin provider",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            configSchema: { type: "object" },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);

    // A managed reconciliation binds the environment to another company after the
    // route guard read an empty binding list. The lease insert then rejects the
    // foreign-company binding with the 403 `environment_company_mismatch`.
    const otherCompanyId = randomUUID();
    await db.insert(companies).values({
      id: otherCompanyId,
      name: "Other Co",
      issuePrefix: "OTB",
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(builtInManagedResources).values({
      companyId: otherCompanyId,
      bundleKey: "managed-environment",
      resourceKind: "environment",
      resourceKey: "managed-sandbox",
      resourceId: environment.id,
      stockVersion: "1",
      stockHash: "hash",
    });

    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentDestroyLease"] })),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "plugin-lease-1",
            metadata: { provider: "fake-plugin", image: "fake:test", reuseLease: false },
          };
        }
        if (method === "environmentDestroyLease") {
          return { providerLeaseId: "plugin-lease-1", state: "destroyed" };
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    await expect(
      runtimeWithPlugin.acquireRunLease({
        companyId,
        environment,
        issueId: null,
        heartbeatRunId: runId,
        persistedExecutionWorkspace: null,
        assertCompanyBinding: true,
      }),
    ).rejects.toMatchObject({
      status: 403,
      details: { code: "environment_company_mismatch" },
    });

    // The acquire records the durable pending-cleanup row before the teardown,
    // so the row lands while the database is proven reachable. The successful
    // teardown then releases the row to the terminal `expired` state, so no
    // active or pending_cleanup row remains for the orphan.
    const leaseRows = await db
      .select()
      .from(environmentLeases)
      .where(eq(environmentLeases.environmentId, environment.id));
    expect(leaseRows).toHaveLength(1);
    expect(leaseRows[0]?.status).toBe("expired");
    expect(leaseRows[0]?.cleanupStatus).toBe("success");
    expect(leaseRows[0]?.metadata?.remoteExecutionTermination).toMatchObject({
      companyId, runId, leaseId: leaseRows[0]!.id, providerLeaseId: "plugin-lease-1", state: "destroyed",
    });

    // The acquire provisioned the remote plugin sandbox, so it destroys the
    // sandbox on the rejection. Without this teardown the rejected insert leaks a
    // live sandbox that no lease row tracks.
    const destroyCalls = (workerManager.call as unknown as ReturnType<typeof vi.fn>).mock.calls.filter(
      (callArgs) => callArgs[1] === "environmentDestroyLease",
    );
    expect(destroyCalls).toHaveLength(1);
    expect(destroyCalls[0]?.[2]).toMatchObject({ providerLeaseId: "plugin-lease-1" });

    await environmentService(db).update(environment.id, { driver: "local", config: {} });
  });

  it("records a durable pending-cleanup lease when the built-in teardown fails after a foreign-company rejection", async () => {
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Foreign-bound Fake Sandbox Teardown Fail",
      config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
    });
    const otherCompanyId = randomUUID();
    await db.insert(companies).values({
      id: otherCompanyId,
      name: "Other Co Built-in",
      issuePrefix: "OTC",
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(builtInManagedResources).values({
      companyId: otherCompanyId,
      bundleKey: "managed-environment",
      resourceKind: "environment",
      resourceKey: "managed-sandbox",
      resourceId: environment.id,
      stockVersion: "1",
      stockHash: "hash",
    });

    // The remote teardown fails, so the acquire cannot release the sandbox
    // directly. It must record a durable pending-cleanup row instead.
    const destroySpy = vi
      .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
      .mockRejectedValue(new Error("teardown failed"));
    try {
      await expect(
        runtime.acquireRunLease({
          companyId,
          environment,
          issueId: null,
          heartbeatRunId: runId,
          persistedExecutionWorkspace: null,
          assertCompanyBinding: true,
        }),
      ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } });

      expect(destroySpy).toHaveBeenCalledTimes(1);
      // The failed teardown left a durable pending-cleanup row that carries the
      // provider lease id for a later sweep.
      const rows = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.environmentId, environment.id));
      expect(rows).toHaveLength(1);
      expect(rows[0]?.status).toBe("pending_cleanup");
      expect(rows[0]?.cleanupStatus).toBe("failed");
      expect(rows[0]?.providerLeaseId).toMatch(new RegExp(`^sandbox://fake/${runId}/[0-9a-f-]{36}$`));
    } finally {
      destroySpy.mockRestore();
    }
  });

  it("records the durable pending-cleanup row before it runs the inline teardown", async () => {
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Foreign-bound Fake Sandbox Write First",
      config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
    });
    const otherCompanyId = randomUUID();
    await db.insert(companies).values({
      id: otherCompanyId,
      name: "Other Co Write First",
      issuePrefix: "OWF",
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(builtInManagedResources).values({
      companyId: otherCompanyId,
      bundleKey: "managed-environment",
      resourceKind: "environment",
      resourceKey: "managed-sandbox",
      resourceId: environment.id,
      stockVersion: "1",
      stockHash: "hash",
    });

    // The teardown reads the lease table when it runs. A durable pending_cleanup
    // row must already exist, which proves the acquire records the row before the
    // teardown. This ordering closes the window the finding describes: the durable
    // write lands while the database is proven reachable, before the teardown RPC
    // that a database outage could otherwise interrupt.
    let pendingCleanupRowsAtTeardown = -1;
    const destroySpy = vi
      .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
      .mockImplementation(async () => {
        const rows = await db
          .select()
          .from(environmentLeases)
          .where(eq(environmentLeases.environmentId, environment.id));
        pendingCleanupRowsAtTeardown = rows.filter((row) => row.status === "pending_cleanup").length;
      });
    try {
      await expect(
        runtime.acquireRunLease({
          companyId,
          environment,
          issueId: null,
          heartbeatRunId: runId,
          persistedExecutionWorkspace: null,
          assertCompanyBinding: true,
        }),
      ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } });

      // The durable row already tracked the orphan when the teardown ran.
      expect(pendingCleanupRowsAtTeardown).toBe(1);
      expect(destroySpy).toHaveBeenCalledTimes(1);

      // The teardown succeeded, so the acquire released the row to the terminal
      // `expired` state and left no pending_cleanup row.
      const rows = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.environmentId, environment.id));
      expect(rows).toHaveLength(1);
      expect(rows[0]?.status).toBe("expired");
      expect(rows[0]?.cleanupStatus).toBe("success");
    } finally {
      destroySpy.mockRestore();
    }
  });

  it("leaves no orphan and no error when the durable write fails but the teardown succeeds", async () => {
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Foreign-bound Fake Sandbox Write Fail Teardown Ok",
      config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
    });
    const otherCompanyId = randomUUID();
    await db.insert(companies).values({
      id: otherCompanyId,
      name: "Other Co Write Fail",
      issuePrefix: "OWK",
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(builtInManagedResources).values({
      companyId: otherCompanyId,
      bundleKey: "managed-environment",
      resourceKind: "environment",
      resourceKey: "managed-sandbox",
      resourceId: environment.id,
      stockVersion: "1",
      stockHash: "hash",
    });

    // Force every durable pending-cleanup write to fail, but let the teardown
    // succeed. A successful teardown removes the orphan, so the acquire needs no
    // durable row and raises no orphan-cleanup write error. The write-first order
    // never turns a clean teardown into a failure.
    const realEnvironmentService = environmentsModule.environmentService;
    const factorySpy = vi
      .spyOn(environmentsModule, "environmentService")
      .mockImplementation((database: Parameters<typeof realEnvironmentService>[0]) => {
        const real = realEnvironmentService(database);
        return {
          ...real,
          insertPendingCleanupLease: () =>
            Promise.reject(new Error("pending-cleanup write failed; database down")),
        };
      });
    const destroySpy = vi
      .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
      .mockResolvedValue(undefined as never);
    try {
      // Set the retry backoff to zero, so the failed write retries never slow the
      // test.
      const runtime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 });
      const rejection = await runtime
        .acquireRunLease({
          companyId,
          environment,
          issueId: null,
          heartbeatRunId: runId,
          persistedExecutionWorkspace: null,
          assertCompanyBinding: true,
        })
        .then(
          () => {
            throw new Error("acquireRunLease resolved but must reject");
          },
          (error: unknown) => error,
        );

      // The rejection is the original company mismatch, not an orphan-cleanup
      // write error, because the teardown removed the orphan.
      expect(rejection).not.toBeInstanceOf(SandboxOrphanCleanupWriteError);
      expect(rejection).toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } });
      expect(destroySpy).toHaveBeenCalledTimes(1);

      // No orphan remains and no durable row was needed, so the lease table is
      // empty for the environment.
      const rows = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.environmentId, environment.id));
      expect(rows).toHaveLength(0);

      // The buffer holds no orphan, so a flush recovers nothing.
      const flushed = await runtime.flushDeferredOrphanCleanups();
      expect(flushed).toEqual({ recovered: 0, pending: 0 });
    } finally {
      destroySpy.mockRestore();
      factorySpy.mockRestore();
    }
  });

  it("records a durable pending-cleanup lease when the plugin teardown fails after a foreign-company rejection", async () => {
    const pluginId = randomUUID();
    const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
    const pluginConfig = { provider: "fake-plugin", image: "fake:test", reuseLease: false };
    const environment = {
      ...baseEnvironment,
      name: "Foreign-bound Plugin Sandbox Teardown Fail",
      driver: "sandbox",
      config: pluginConfig,
    };
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: pluginConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "paperclip.fake-plugin-sandbox-provider",
      packageName: "@paperclipai/plugin-fake-sandbox",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "paperclip.fake-plugin-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Fake Plugin Sandbox Provider",
        description: "Test fake plugin provider",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            configSchema: { type: "object" },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);
    const otherCompanyId = randomUUID();
    await db.insert(companies).values({
      id: otherCompanyId,
      name: "Other Co Plugin",
      issuePrefix: "OTD",
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(builtInManagedResources).values({
      companyId: otherCompanyId,
      bundleKey: "managed-environment",
      resourceKind: "environment",
      resourceKey: "managed-sandbox",
      resourceId: environment.id,
      stockVersion: "1",
      stockHash: "hash",
    });

    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentDestroyLease"] })),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "plugin-lease-2",
            metadata: { provider: "fake-plugin", image: "fake:test", reuseLease: false },
          };
        }
        if (method === "environmentDestroyLease") {
          throw new Error("destroy failed");
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    await expect(
      runtimeWithPlugin.acquireRunLease({
        companyId,
        environment,
        issueId: null,
        heartbeatRunId: runId,
        persistedExecutionWorkspace: null,
        assertCompanyBinding: true,
      }),
    ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } });

    // The failed plugin teardown left a durable pending-cleanup row that carries
    // the provider lease id for a later sweep.
    const rows = await db
      .select()
      .from(environmentLeases)
      .where(eq(environmentLeases.environmentId, environment.id));
    expect(rows).toHaveLength(1);
    expect(rows[0]?.status).toBe("pending_cleanup");
    expect(rows[0]?.cleanupStatus).toBe("failed");
    expect(rows[0]?.providerLeaseId).toBe("plugin-lease-2");

    await environmentService(db).update(environment.id, { driver: "local", config: {} });
  });

  it("throws a SandboxOrphanCleanupWriteError when the built-in durable pending-cleanup write also fails", async () => {
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Foreign-bound Fake Sandbox Cleanup Write Fail",
      config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
    });
    const otherCompanyId = randomUUID();
    await db.insert(companies).values({
      id: otherCompanyId,
      name: "Other Co Cleanup Write",
      issuePrefix: "OCW",
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(builtInManagedResources).values({
      companyId: otherCompanyId,
      bundleKey: "managed-environment",
      resourceKind: "environment",
      resourceKey: "managed-sandbox",
      resourceId: environment.id,
      stockVersion: "1",
      stockHash: "hash",
    });

    // The remote teardown fails, so the acquire tries to record a durable
    // pending-cleanup row instead.
    const destroySpy = vi
      .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
      .mockRejectedValue(new Error("teardown failed"));
    // Force the durable pending-cleanup write to fail on every attempt. The
    // write retries a few times, so reject the insert each time and let the
    // primary company-bound acquire reject for real.
    const failingInsert = vi
      .fn<Parameters<ReturnType<typeof environmentService>["insertPendingCleanupLease"]>, Promise<never>>()
      .mockRejectedValue(new Error("pending-cleanup write failed"));
    const realEnvironmentService = environmentsModule.environmentService;
    const factorySpy = vi
      .spyOn(environmentsModule, "environmentService")
      .mockImplementation((database: Parameters<typeof realEnvironmentService>[0]) => {
        const real = realEnvironmentService(database);
        return {
          ...real,
          insertPendingCleanupLease: failingInsert,
        };
      });
    // The durable database write fails, so the only durable handle left is the
    // error log. Capture it to prove the leaked sandbox stays discoverable.
    const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger);
    try {
      // Set the retry backoff to zero, so the retries never slow the test.
      const runtimeWithFailingCleanup = environmentRuntimeService(db, {
        pendingCleanupWriteBackoffMs: 0,
      });
      const rejection = await runtimeWithFailingCleanup
        .acquireRunLease({
          companyId,
          environment,
          issueId: null,
          heartbeatRunId: runId,
          persistedExecutionWorkspace: null,
          assertCompanyBinding: true,
        })
        .then(
          () => {
            throw new Error("acquireRunLease resolved but must reject");
          },
          (error: unknown) => error,
        );

      // The failed durable write is not swallowed: the acquire throws a
      // SandboxOrphanCleanupWriteError that keeps the original 403 rejection as
      // its cause.
      expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError);
      expect(rejection).toMatchObject({ provider: "fake" });
      expect((rejection as SandboxOrphanCleanupWriteError).cause).toMatchObject({
        status: 403,
        details: { code: "environment_company_mismatch" },
      });
      expect(destroySpy).toHaveBeenCalledTimes(1);
      // The write retried the durable insert before it gave up, so more than one
      // attempt ran.
      expect(failingInsert.mock.calls.length).toBeGreaterThan(1);
      // The durable write failed before it created a row, so no lease row exists.
      const rows = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.environmentId, environment.id));
      expect(rows).toHaveLength(0);
      // The durable log preserves the provider identifiers, so an operator keeps
      // a handle to find and tear the leaked sandbox down by hand.
      const cleanupLog = logSpy.mock.calls.find(
        ([fields]) =>
          (fields as { errorKind?: string } | undefined)?.errorKind ===
          "sandbox_orphan_cleanup_write_failed",
      );
      expect(cleanupLog).toBeDefined();
      expect(cleanupLog?.[0]).toMatchObject({
        provider: "fake",
        companyId,
        environmentId: environment.id,
      });
      // The log never carries the caught exception, so no credential leaks.
      expect(cleanupLog?.[0]).not.toHaveProperty("cause");
      expect(cleanupLog?.[0]).not.toHaveProperty("cleanupWriteError");
    } finally {
      logSpy.mockRestore();
      factorySpy.mockRestore();
      destroySpy.mockRestore();
    }
  });

  it("recovers cleanup state when a retry of the durable pending-cleanup write succeeds", async () => {
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Foreign-bound Fake Sandbox Cleanup Write Retry",
      config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
    });
    const otherCompanyId = randomUUID();
    await db.insert(companies).values({
      id: otherCompanyId,
      name: "Other Co Cleanup Retry",
      issuePrefix: "OCR",
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(builtInManagedResources).values({
      companyId: otherCompanyId,
      bundleKey: "managed-environment",
      resourceKind: "environment",
      resourceKey: "managed-sandbox",
      resourceId: environment.id,
      stockVersion: "1",
      stockHash: "hash",
    });

    // The remote teardown fails, so the acquire tries to record a durable
    // pending-cleanup row instead.
    const destroySpy = vi
      .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
      .mockRejectedValue(new Error("teardown failed"));
    // The first durable write attempt fails, so the acquire retries. The second
    // attempt delegates to the real insert, so a durable pending_cleanup row
    // lands and a later sweep can find the orphan.
    let insertAttempts = 0;
    const realEnvironmentService = environmentsModule.environmentService;
    const factorySpy = vi
      .spyOn(environmentsModule, "environmentService")
      .mockImplementation((database: Parameters<typeof realEnvironmentService>[0]) => {
        const real = realEnvironmentService(database);
        return {
          ...real,
          insertPendingCleanupLease: (
            input: Parameters<typeof real.insertPendingCleanupLease>[0],
          ) => {
            insertAttempts += 1;
            if (insertAttempts === 1) {
              return Promise.reject(new Error("pending-cleanup write failed once"));
            }
            return real.insertPendingCleanupLease(input);
          },
        };
      });
    try {
      // Set the retry backoff to zero, so the retry never slows the test.
      const runtimeWithRetry = environmentRuntimeService(db, {
        pendingCleanupWriteBackoffMs: 0,
      });
      const rejection = await runtimeWithRetry
        .acquireRunLease({
          companyId,
          environment,
          issueId: null,
          heartbeatRunId: runId,
          persistedExecutionWorkspace: null,
          assertCompanyBinding: true,
        })
        .then(
          () => {
            throw new Error("acquireRunLease resolved but must reject");
          },
          (error: unknown) => error,
        );

      // The retry landed the durable row, so the acquire rejects with the
      // original company-mismatch rejection, not a SandboxOrphanCleanupWriteError.
      expect(rejection).not.toBeInstanceOf(SandboxOrphanCleanupWriteError);
      expect(rejection).toMatchObject({
        status: 403,
        details: { code: "environment_company_mismatch" },
      });
      expect(destroySpy).toHaveBeenCalledTimes(1);
      // The first attempt failed and the second succeeded, so the write ran twice.
      expect(insertAttempts).toBe(2);
      // A durable pending_cleanup lease row now tracks the orphan, so a sweep can
      // find and release the leaked sandbox.
      const rows = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.environmentId, environment.id));
      expect(rows).toHaveLength(1);
      expect(rows[0]?.status).toBe("pending_cleanup");
      expect(rows[0]?.cleanupStatus).toBe("failed");
      expect(rows[0]?.failureReason).toBe("acquire_rejected_teardown_failed");
    } finally {
      factorySpy.mockRestore();
      destroySpy.mockRestore();
    }
  });

  it("buffers the orphan in-process and a later sweep flush lands the durable row after the database recovers", async () => {
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Foreign-bound Fake Sandbox Cleanup Buffer Flush",
      config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
    });
    const otherCompanyId = randomUUID();
    await db.insert(companies).values({
      id: otherCompanyId,
      name: "Other Co Cleanup Buffer",
      issuePrefix: "OCB",
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(builtInManagedResources).values({
      companyId: otherCompanyId,
      bundleKey: "managed-environment",
      resourceKind: "environment",
      resourceKey: "managed-sandbox",
      resourceId: environment.id,
      stockVersion: "1",
      stockHash: "hash",
    });

    // The remote teardown fails, so the acquire tries to record a durable
    // pending-cleanup row instead.
    const destroySpy = vi
      .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
      .mockRejectedValue(new Error("teardown failed"));
    // The database is down, so every synchronous pending-cleanup write rejects.
    // The flag flips to false when the database recovers, so a later flush lands
    // the durable row.
    let databaseDown = true;
    const realEnvironmentService = environmentsModule.environmentService;
    const factorySpy = vi
      .spyOn(environmentsModule, "environmentService")
      .mockImplementation((database: Parameters<typeof realEnvironmentService>[0]) => {
        const real = realEnvironmentService(database);
        return {
          ...real,
          insertPendingCleanupLease: (
            input: Parameters<typeof real.insertPendingCleanupLease>[0],
          ) => {
            if (databaseDown) {
              return Promise.reject(new Error("pending-cleanup write failed; database down"));
            }
            return real.insertPendingCleanupLease(input);
          },
        };
      });
    const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger);
    try {
      // Set the retry backoff to zero, so the retries never slow the test.
      const runtime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 });
      const rejection = await runtime
        .acquireRunLease({
          companyId,
          environment,
          issueId: null,
          heartbeatRunId: runId,
          persistedExecutionWorkspace: null,
          assertCompanyBinding: true,
        })
        .then(
          () => {
            throw new Error("acquireRunLease resolved but must reject");
          },
          (error: unknown) => error,
        );

      // The synchronous write failed on every attempt, so the acquire still
      // throws the orphan-cleanup write error that keeps the original rejection.
      expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError);
      // The database was down, so no durable row exists yet.
      let rows = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.environmentId, environment.id));
      expect(rows).toHaveLength(0);
      // The acquire buffered the orphan in-process, so the log records the buffer.
      const bufferedLog = logSpy.mock.calls.find(
        ([fields]) =>
          (fields as { errorKind?: string } | undefined)?.errorKind ===
          "sandbox_orphan_cleanup_write_failed",
      );
      expect(bufferedLog?.[0]).toMatchObject({ buffered: true });

      // The database recovers, so the next cleanup-sweep flush lands the row.
      databaseDown = false;
      const flushed = await runtime.flushDeferredOrphanCleanups();
      expect(flushed).toEqual({ recovered: 1, pending: 0 });

      // A durable pending_cleanup row now tracks the orphan, so a sweep finds and
      // releases the leaked sandbox.
      rows = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.environmentId, environment.id));
      expect(rows).toHaveLength(1);
      expect(rows[0]?.status).toBe("pending_cleanup");
      expect(rows[0]?.cleanupStatus).toBe("failed");
      expect(rows[0]?.failureReason).toBe("acquire_rejected_teardown_failed");
      expect(rows[0]?.providerLeaseId).toBeTruthy();

      // The buffer is empty now, so a second flush inserts nothing.
      const second = await runtime.flushDeferredOrphanCleanups();
      expect(second).toEqual({ recovered: 0, pending: 0 });
      const rowsAfter = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.environmentId, environment.id));
      expect(rowsAfter).toHaveLength(1);
    } finally {
      logSpy.mockRestore();
      factorySpy.mockRestore();
      destroySpy.mockRestore();
    }
  });

  it("persists the orphan to the durable spool so a restart recovers and lands the durable row", async () => {
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Foreign-bound Fake Sandbox Cleanup Spool Restart",
      config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
    });
    const otherCompanyId = randomUUID();
    await db.insert(companies).values({
      id: otherCompanyId,
      name: "Other Co Cleanup Spool Restart",
      issuePrefix: "OCS",
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(builtInManagedResources).values({
      companyId: otherCompanyId,
      bundleKey: "managed-environment",
      resourceKind: "environment",
      resourceKey: "managed-sandbox",
      resourceId: environment.id,
      stockVersion: "1",
      stockHash: "hash",
    });

    // The remote teardown fails, so the acquire tries to record a durable
    // pending-cleanup row instead.
    const destroySpy = vi
      .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
      .mockRejectedValue(new Error("teardown failed"));
    // The database is down, so every synchronous pending-cleanup write rejects.
    // The flag flips to false to simulate the database recovering after a
    // restart.
    let databaseDown = true;
    const realEnvironmentService = environmentsModule.environmentService;
    const factorySpy = vi
      .spyOn(environmentsModule, "environmentService")
      .mockImplementation((database: Parameters<typeof realEnvironmentService>[0]) => {
        const real = realEnvironmentService(database);
        return {
          ...real,
          insertPendingCleanupLease: (
            input: Parameters<typeof real.insertPendingCleanupLease>[0],
          ) => {
            if (databaseDown) {
              return Promise.reject(new Error("pending-cleanup write failed; database down"));
            }
            return real.insertPendingCleanupLease(input);
          },
        };
      });
    const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger);
    const spoolDir = process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR!;
    try {
      // The first process acquires, fails every synchronous write, and persists
      // the orphan to the durable spool. The retry backoff is zero, so the test
      // never waits.
      const firstProcessRuntime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 });
      const rejection = await firstProcessRuntime
        .acquireRunLease({
          companyId,
          environment,
          issueId: null,
          heartbeatRunId: runId,
          persistedExecutionWorkspace: null,
          assertCompanyBinding: true,
        })
        .then(
          () => {
            throw new Error("acquireRunLease resolved but must reject");
          },
          (error: unknown) => error,
        );
      expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError);

      // The database was down, so no durable row exists yet.
      let rows = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.environmentId, environment.id));
      expect(rows).toHaveLength(0);

      // The acquire persisted the orphan to the durable spool, so a file waits on
      // the disk and the log records the durable persist.
      const spoolFiles = (await readdir(spoolDir)).filter((name) => name.endsWith(".json"));
      expect(spoolFiles).toHaveLength(1);
      const persistedLog = logSpy.mock.calls.find(
        ([fields]) =>
          (fields as { errorKind?: string } | undefined)?.errorKind ===
          "sandbox_orphan_cleanup_write_failed",
      );
      expect(persistedLog?.[0]).toMatchObject({ persisted: true });

      // The process restarts. A fresh runtime keeps no in-process buffer, so only
      // the durable spool carries the orphan. The database recovers, so the first
      // cleanup-sweep flush loads the spool and lands the durable row.
      databaseDown = false;
      const restartedRuntime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 });
      const flushed = await restartedRuntime.flushDeferredOrphanCleanups();
      expect(flushed).toEqual({ recovered: 1, pending: 0 });

      rows = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.environmentId, environment.id));
      expect(rows).toHaveLength(1);
      expect(rows[0]?.status).toBe("pending_cleanup");
      expect(rows[0]?.failureReason).toBe("acquire_rejected_teardown_failed");
      expect(rows[0]?.providerLeaseId).toBeTruthy();

      // The flush removed the spooled copy after the row landed, so a second
      // restart never re-inserts a duplicate.
      const spoolFilesAfter = (await readdir(spoolDir)).filter((name) => name.endsWith(".json"));
      expect(spoolFilesAfter).toHaveLength(0);
      const secondRestartRuntime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 });
      const secondFlush = await secondRestartRuntime.flushDeferredOrphanCleanups();
      expect(secondFlush).toEqual({ recovered: 0, pending: 0 });
      const rowsAfter = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.environmentId, environment.id));
      expect(rowsAfter).toHaveLength(1);
    } finally {
      logSpy.mockRestore();
      factorySpy.mockRestore();
      destroySpy.mockRestore();
    }
  });

  it("buffers the orphan in-process when the durable spool write also fails", async () => {
    const idleBefore = idleWorkSnapshot().active;
    let releaseFlush!: () => void;
    let enteredFlush!: () => void;
    const flushGate = new Promise<void>(resolve => { releaseFlush = resolve; });
    const flushEntered = new Promise<void>(resolve => { enteredFlush = resolve; });
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Foreign-bound Fake Sandbox Cleanup Spool Unwritable",
      config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
    });
    const otherCompanyId = randomUUID();
    await db.insert(companies).values({
      id: otherCompanyId,
      name: "Other Co Cleanup Spool Unwritable",
      issuePrefix: "OCU",
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(builtInManagedResources).values({
      companyId: otherCompanyId,
      bundleKey: "managed-environment",
      resourceKind: "environment",
      resourceKey: "managed-sandbox",
      resourceId: environment.id,
      stockVersion: "1",
      stockHash: "hash",
    });

    const destroySpy = vi
      .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
      .mockRejectedValue(new Error("teardown failed"));
    let databaseDown = true;
    const realEnvironmentService = environmentsModule.environmentService;
    const factorySpy = vi
      .spyOn(environmentsModule, "environmentService")
      .mockImplementation((database: Parameters<typeof realEnvironmentService>[0]) => {
        const real = realEnvironmentService(database);
        return {
          ...real,
          insertPendingCleanupLease: (
            input: Parameters<typeof real.insertPendingCleanupLease>[0],
          ) => {
            if (databaseDown) {
              return Promise.reject(new Error("pending-cleanup write failed; database down"));
            }
            enteredFlush();
            return flushGate.then(() => real.insertPendingCleanupLease(input));
          },
        };
      });
    const logSpy = vi.spyOn(logger, "error").mockImplementation(() => logger);
    try {
      // Point the spool at an unwritable path, so the durable spool write fails
      // like a full or read-only disk. The acquire must fall back to the
      // in-process buffer and never lose the orphan silently.
      const unwritableSpool = {
        append: () => Promise.resolve(false),
        load: () => Promise.resolve([]),
        remove: () => Promise.resolve(),
      };
      const runtime = environmentRuntimeService(db, {
        pendingCleanupWriteBackoffMs: 0,
        orphanCleanupSpool: unwritableSpool,
      });
      await runtime
        .acquireRunLease({
          companyId,
          environment,
          issueId: null,
          heartbeatRunId: runId,
          persistedExecutionWorkspace: null,
          assertCompanyBinding: true,
        })
        .then(
          () => {
            throw new Error("acquireRunLease resolved but must reject");
          },
          () => undefined,
        );

      // The spool write failed, so the log records the in-process-only fallback.
      const fallbackLog = logSpy.mock.calls.find(
        ([fields]) =>
          (fields as { errorKind?: string } | undefined)?.errorKind ===
          "sandbox_orphan_cleanup_write_failed",
      );
      expect(fallbackLog?.[0]).toMatchObject({ persisted: false, buffered: true });
      expect(idleWorkSnapshot().active).toBe(idleBefore + 1);

      // The same runtime still keeps the orphan in-process, so a flush after the
      // database recovers lands the durable row.
      databaseDown = false;
      const flushing = runtime.flushDeferredOrphanCleanups();
      await flushEntered;
      // The batch was spliced out of the queue, but the DB does not own it yet.
      expect(idleWorkSnapshot().active).toBe(idleBefore + 1);
      releaseFlush();
      const flushed = await flushing;
      expect(idleWorkSnapshot().active).toBe(idleBefore);
      expect(flushed).toEqual({ recovered: 1, pending: 0 });
      const rows = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.environmentId, environment.id));
      expect(rows).toHaveLength(1);
      expect(rows[0]?.status).toBe("pending_cleanup");
    } finally {
      releaseFlush();
      logSpy.mockRestore();
      factorySpy.mockRestore();
      destroySpy.mockRestore();
    }
  });

  it("keeps the buffered orphan when the flush write still fails, then recovers on a later flush", async () => {
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Foreign-bound Fake Sandbox Cleanup Buffer Requeue",
      config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
    });
    const otherCompanyId = randomUUID();
    await db.insert(companies).values({
      id: otherCompanyId,
      name: "Other Co Cleanup Requeue",
      issuePrefix: "OCQ",
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(builtInManagedResources).values({
      companyId: otherCompanyId,
      bundleKey: "managed-environment",
      resourceKind: "environment",
      resourceKey: "managed-sandbox",
      resourceId: environment.id,
      stockVersion: "1",
      stockHash: "hash",
    });

    const destroySpy = vi
      .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
      .mockRejectedValue(new Error("teardown failed"));
    let databaseDown = true;
    const realEnvironmentService = environmentsModule.environmentService;
    const factorySpy = vi
      .spyOn(environmentsModule, "environmentService")
      .mockImplementation((database: Parameters<typeof realEnvironmentService>[0]) => {
        const real = realEnvironmentService(database);
        return {
          ...real,
          insertPendingCleanupLease: (
            input: Parameters<typeof real.insertPendingCleanupLease>[0],
          ) => {
            if (databaseDown) {
              return Promise.reject(new Error("pending-cleanup write failed; database down"));
            }
            return real.insertPendingCleanupLease(input);
          },
        };
      });
    const errorLogSpy = vi.spyOn(logger, "error").mockImplementation(() => logger);
    const warnLogSpy = vi.spyOn(logger, "warn").mockImplementation(() => logger);
    try {
      const runtime = environmentRuntimeService(db, { pendingCleanupWriteBackoffMs: 0 });
      await runtime
        .acquireRunLease({
          companyId,
          environment,
          issueId: null,
          heartbeatRunId: runId,
          persistedExecutionWorkspace: null,
          assertCompanyBinding: true,
        })
        .then(
          () => {
            throw new Error("acquireRunLease resolved but must reject");
          },
          () => undefined,
        );

      // The database is still down, so the flush re-queues the orphan instead of
      // losing it. The buffer keeps the record for a later tick.
      const firstFlush = await runtime.flushDeferredOrphanCleanups();
      expect(firstFlush).toEqual({ recovered: 0, pending: 1 });
      const rowsWhileDown = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.environmentId, environment.id));
      expect(rowsWhileDown).toHaveLength(0);
      // The flush logs the re-queue and never carries the caught write exception.
      // The sync-retry path also warns with the same error kind, so match on the
      // `requeued` field that only the flush warn carries.
      const requeueLog = warnLogSpy.mock.calls.find(
        ([fields]) => (fields as { requeued?: boolean } | undefined)?.requeued === true,
      );
      expect(requeueLog).toBeDefined();
      expect(requeueLog?.[0]).toMatchObject({
        errorKind: "sandbox_orphan_cleanup_write_failed",
        requeued: true,
      });
      expect(requeueLog?.[0]).not.toHaveProperty("cause");
      expect(requeueLog?.[0]).not.toHaveProperty("cleanupWriteError");

      // The database recovers, so the next flush lands the durable row exactly
      // once from the still-buffered record.
      databaseDown = false;
      const secondFlush = await runtime.flushDeferredOrphanCleanups();
      expect(secondFlush).toEqual({ recovered: 1, pending: 0 });
      const rowsAfter = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.environmentId, environment.id));
      expect(rowsAfter).toHaveLength(1);
      expect(rowsAfter[0]?.status).toBe("pending_cleanup");
    } finally {
      warnLogSpy.mockRestore();
      errorLogSpy.mockRestore();
      factorySpy.mockRestore();
      destroySpy.mockRestore();
    }
  });

  it("throws a SandboxOrphanCleanupWriteError when the plugin durable pending-cleanup write also fails", async () => {
    const pluginId = randomUUID();
    const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
    const pluginConfig = { provider: "fake-plugin", image: "fake:test", reuseLease: false };
    const environment = {
      ...baseEnvironment,
      name: "Foreign-bound Plugin Sandbox Cleanup Write Fail",
      driver: "sandbox",
      config: pluginConfig,
    };
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: pluginConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "paperclip.fake-plugin-sandbox-provider",
      packageName: "@paperclipai/plugin-fake-sandbox",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "paperclip.fake-plugin-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Fake Plugin Sandbox Provider",
        description: "Test fake plugin provider",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            configSchema: { type: "object" },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);
    const otherCompanyId = randomUUID();
    await db.insert(companies).values({
      id: otherCompanyId,
      name: "Other Co Plugin Cleanup Write",
      issuePrefix: "OPW",
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(builtInManagedResources).values({
      companyId: otherCompanyId,
      bundleKey: "managed-environment",
      resourceKind: "environment",
      resourceKey: "managed-sandbox",
      resourceId: environment.id,
      stockVersion: "1",
      stockHash: "hash",
    });

    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentDestroyLease"] })),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "plugin-lease-write-fail",
            metadata: { provider: "fake-plugin", image: "fake:test", reuseLease: false },
          };
        }
        if (method === "environmentDestroyLease") {
          throw new Error("destroy failed");
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
    } as unknown as PluginWorkerManager;

    const realEnvironmentService = environmentsModule.environmentService;
    const factorySpy = vi
      .spyOn(environmentsModule, "environmentService")
      .mockImplementation((database: Parameters<typeof realEnvironmentService>[0]) => {
        const real = realEnvironmentService(database);
        return {
          ...real,
          insertPendingCleanupLease: () =>
            Promise.reject(new Error("pending-cleanup write failed")),
        };
      });
    try {
      const runtimeWithPlugin = environmentRuntimeService(db, {
        pluginWorkerManager: workerManager,
        pendingCleanupWriteBackoffMs: 0,
      });
      const rejection = await runtimeWithPlugin
        .acquireRunLease({
          companyId,
          environment,
          issueId: null,
          heartbeatRunId: runId,
          persistedExecutionWorkspace: null,
          assertCompanyBinding: true,
        })
        .then(
          () => {
            throw new Error("acquireRunLease resolved but must reject");
          },
          (error: unknown) => error,
        );

      expect(rejection).toBeInstanceOf(SandboxOrphanCleanupWriteError);
      expect(rejection).toMatchObject({ provider: "fake-plugin", providerLeaseId: "plugin-lease-write-fail" });
      expect((rejection as SandboxOrphanCleanupWriteError).cause).toMatchObject({
        status: 403,
        details: { code: "environment_company_mismatch" },
      });
      // The durable write failed before it created a row, so no lease row exists.
      const rows = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.environmentId, environment.id));
      expect(rows).toHaveLength(0);
    } finally {
      factorySpy.mockRestore();
    }

    await environmentService(db).update(environment.id, { driver: "local", config: {} });
  });

  it("records the orphan directly in the pending_cleanup state with one atomic insert", async () => {
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Atomic Pending Cleanup Insert",
      config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
    });

    const lease = await environmentService(db).insertPendingCleanupLease({
      companyId,
      environmentId: environment.id,
      executionWorkspaceId: null,
      issueId: null,
      heartbeatRunId: runId,
      provider: "fake",
      providerLeaseId: "sandbox://fake/atomic-orphan",
      metadata: { provider: "fake", reuseLease: false },
      failureReason: "acquire_rejected_teardown_failed",
    });

    // The returned lease is already in the terminal recovery state. It never
    // passes through the `active` state, so a crash cannot strand the orphan.
    expect(lease.status).toBe("pending_cleanup");
    expect(lease.cleanupStatus).toBe("failed");
    expect(lease.leasePolicy).toBe("ephemeral");
    expect(lease.failureReason).toBe("acquire_rejected_teardown_failed");
    expect(lease.providerLeaseId).toBe("sandbox://fake/atomic-orphan");
    expect(lease.releasedAt).not.toBeNull();

    // One insert created exactly one row, and the sweep reads that row directly.
    const rows = await db
      .select()
      .from(environmentLeases)
      .where(eq(environmentLeases.environmentId, environment.id));
    expect(rows).toHaveLength(1);
    expect(rows[0]?.status).toBe("pending_cleanup");
    expect(rows[0]?.cleanupStatus).toBe("failed");
  });

  it("tears the recorded built-in provider lease down for a pending-cleanup orphan", async () => {
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Cleanup Retry Built-in",
      config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
    });
    const otherCompanyId = randomUUID();
    await db.insert(companies).values({
      id: otherCompanyId,
      name: "Other Co Retry Built-in",
      issuePrefix: "OTE",
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(builtInManagedResources).values({
      companyId: otherCompanyId,
      bundleKey: "managed-environment",
      resourceKind: "environment",
      resourceKey: "managed-sandbox",
      resourceId: environment.id,
      stockVersion: "1",
      stockHash: "hash",
    });

    // The first teardown fails, so the acquire records a pending-cleanup orphan.
    // The sweep then retries the teardown, which succeeds this time.
    const destroySpy = vi
      .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
      .mockRejectedValueOnce(new Error("teardown failed"))
      .mockResolvedValue(undefined);
    try {
      await expect(
        runtime.acquireRunLease({
          companyId,
          environment,
          issueId: null,
          heartbeatRunId: runId,
          persistedExecutionWorkspace: null,
          assertCompanyBinding: true,
        }),
      ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } });

      const orphan = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.environmentId, environment.id))
        .then((r) => r[0]!);
      expect(orphan.status).toBe("pending_cleanup");
      const providerLeaseId = orphan.providerLeaseId;

      // The teardown accepts a null environment and reads the recorded provider
      // lease, so a delete or a provider change never strands it.
      const lease = await environmentService(db).getLeaseById(orphan.id);
      await runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! });
      // The retry called the provider teardown a second time with the orphan
      // provider lease id.
      expect(destroySpy).toHaveBeenCalledTimes(2);
      expect(destroySpy).toHaveBeenLastCalledWith(expect.objectContaining({ providerLeaseId }));
    } finally {
      destroySpy.mockRestore();
    }
  });

  it("throws and keeps the pending-cleanup orphan when the teardown retry fails", async () => {
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Cleanup Retry Scope",
      config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
    });
    const otherCompanyId = randomUUID();
    await db.insert(companies).values({
      id: otherCompanyId,
      name: "Other Co Retry Scope",
      issuePrefix: "OTF",
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(builtInManagedResources).values({
      companyId: otherCompanyId,
      bundleKey: "managed-environment",
      resourceKind: "environment",
      resourceKey: "managed-sandbox",
      resourceId: environment.id,
      stockVersion: "1",
      stockHash: "hash",
    });

    const destroySpy = vi
      .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
      .mockRejectedValue(new Error("teardown failed"));
    try {
      await expect(
        runtime.acquireRunLease({
          companyId,
          environment,
          issueId: null,
          heartbeatRunId: runId,
          persistedExecutionWorkspace: null,
          assertCompanyBinding: true,
        }),
      ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } });

      const orphan = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.environmentId, environment.id))
        .then((r) => r[0]!);
      expect(orphan.status).toBe("pending_cleanup");

      // The teardown retries, but the provider destroy still fails, so the
      // teardown throws. The caller keeps the orphan pending for a later sweep.
      const lease = await environmentService(db).getLeaseById(orphan.id);
      await expect(
        runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! }),
      ).rejects.toThrow();

      const still = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.id, orphan.id))
        .then((r) => r[0]!);
      expect(still.status).toBe("pending_cleanup");
      expect(still.cleanupStatus).toBe("failed");
    } finally {
      destroySpy.mockRestore();
    }
  });

  it("tears the recorded provider down after the environment provider changes", async () => {
    // Ordering: an acquire provisions a sandbox with provider A, the lease
    // insert rejects a foreign-company binding, and the compensating teardown
    // fails, so the acquire records an orphan for provider A. A provider change
    // then re-points the environment before the sweep runs. The sweep must tear
    // the orphan down from the recorded provider metadata, not the current
    // environment provider, so the change cannot strand the teardown.
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Provider Change Vs Orphan",
      config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
    });
    const otherCompanyId = randomUUID();
    await db.insert(companies).values({
      id: otherCompanyId,
      name: "Other Co Provider Change",
      issuePrefix: "OTP",
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(builtInManagedResources).values({
      companyId: otherCompanyId,
      bundleKey: "managed-environment",
      resourceKind: "environment",
      resourceKey: "managed-sandbox",
      resourceId: environment.id,
      stockVersion: "1",
      stockHash: "hash",
    });

    const destroySpy = vi
      .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
      .mockRejectedValueOnce(new Error("teardown failed"))
      .mockResolvedValue(undefined);
    try {
      await expect(
        runtime.acquireRunLease({
          companyId,
          environment,
          issueId: null,
          heartbeatRunId: runId,
          persistedExecutionWorkspace: null,
          assertCompanyBinding: true,
        }),
      ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } });

      const orphan = await db
        .select()
        .from(environmentLeases)
        .where(eq(environmentLeases.environmentId, environment.id))
        .then((r) => r[0]!);
      expect(orphan.status).toBe("pending_cleanup");
      expect(orphan.provider).toBe("fake");
      const providerLeaseId = orphan.providerLeaseId;

      // The provider-target mutation re-points the environment to a different
      // driver. The orphan teardown must not depend on this current config.
      await environmentService(db).update(environment.id, { driver: "local", config: {} });

      // The teardown tears the recorded provider lease down, so it never reads
      // the re-pointed environment provider.
      const lease = await environmentService(db).getLeaseById(orphan.id);
      await runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! });
      expect(destroySpy).toHaveBeenLastCalledWith(expect.objectContaining({ providerLeaseId }));
    } finally {
      destroySpy.mockRestore();
    }
  });

  it("reports the plugin cleanup worker not ready while its worker is down, then ready after it recovers", async () => {
    // A pending_cleanup orphan targets a plugin-backed provider. The sweep must
    // not consume a finite retry attempt while the plugin worker is briefly down.
    // So the readiness probe reports "not ready" for a down worker and "ready"
    // after the worker recovers. A built-in provider has no worker, so it is
    // always ready.
    const pluginId = randomUUID();
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Cleanup Worker Readiness",
      config: { provider: "fake-plugin-ready", image: "fake:test", reuseLease: false },
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "paperclip.fake-plugin-ready-sandbox-provider",
      packageName: "@paperclipai/plugin-fake-ready-sandbox",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "paperclip.fake-plugin-ready-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Fake Plugin Ready Sandbox Provider",
        description: "Test fake plugin provider readiness",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin-ready",
            kind: "sandbox_provider",
            displayName: "Fake Plugin Ready",
            configSchema: { type: "object" },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);

    // The worker is down at first, then recovers on the second probe.
    let workerRunning = false;
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId && workerRunning),
      call: vi.fn(async () => {
        throw new Error("call must not run during a readiness probe");
      }),
    } as unknown as PluginWorkerManager;

    const orphan = await environmentService(db).insertPendingCleanupLease({
      companyId,
      environmentId: environment.id,
      executionWorkspaceId: null,
      issueId: null,
      heartbeatRunId: runId,
      provider: "fake-plugin-ready",
      providerLeaseId: "plugin-lease-readiness",
      metadata: { provider: "fake-plugin-ready", driver: "sandbox", reuseLease: false },
      failureReason: "acquire_rejected_teardown_failed",
    });
    const lease = (await environmentService(db).getLeaseById(orphan.id))!;

    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    // A down plugin worker is the transient restart window, so the probe reports
    // not ready. The sweep skips the lease without a claim, so no attempt burns.
    await expect(
      runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }),
    ).resolves.toBe(false);
    // The probe never calls the worker; it only checks the live worker state.
    expect((workerManager.call as ReturnType<typeof vi.fn>)).not.toHaveBeenCalled();

    // The worker recovers, so the probe now reports ready and the sweep proceeds.
    workerRunning = true;
    await expect(
      runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }),
    ).resolves.toBe(true);

    // A built-in provider has no plugin worker, so its orphan is always ready.
    const builtinOrphan = await environmentService(db).insertPendingCleanupLease({
      companyId,
      environmentId: environment.id,
      executionWorkspaceId: null,
      issueId: null,
      heartbeatRunId: runId,
      provider: "fake",
      providerLeaseId: "builtin-lease-readiness",
      metadata: { provider: "fake", driver: "sandbox", reuseLease: false },
      failureReason: "acquire_rejected_teardown_failed",
    });
    const builtinLease = (await environmentService(db).getLeaseById(builtinOrphan.id))!;
    await expect(
      runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease: builtinLease }),
    ).resolves.toBe(true);
  });

  it("reports the plugin cleanup provider not ready while the plugin is not ready, then ready after it recovers", async () => {
    // A pending_cleanup orphan targets a plugin-backed provider. A plugin reload
    // or a plugin reinstall moves the plugin through a "not ready" status. The
    // sweep must not consume a finite retry attempt in that transient window. So
    // the readiness probe reports "not ready" while the plugin status is not
    // "ready", even when its worker runs, and "ready" after the plugin recovers.
    const pluginId = randomUUID();
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Cleanup Plugin Readiness",
      config: { provider: "fake-plugin-reload", image: "fake:test", reuseLease: false },
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "paperclip.fake-plugin-reload-sandbox-provider",
      packageName: "@paperclipai/plugin-fake-reload-sandbox",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "paperclip.fake-plugin-reload-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Fake Plugin Reload Sandbox Provider",
        description: "Test fake plugin provider reload readiness",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin-reload",
            kind: "sandbox_provider",
            displayName: "Fake Plugin Reload",
            configSchema: { type: "object" },
          },
        ],
      },
      // The plugin starts in a not-ready status, as during a reload.
      status: "installing",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);

    // The worker runs the whole time, so the probe result depends on the plugin
    // status alone, not on a down worker.
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async () => {
        throw new Error("call must not run during a readiness probe");
      }),
    } as unknown as PluginWorkerManager;

    const orphan = await environmentService(db).insertPendingCleanupLease({
      companyId,
      environmentId: environment.id,
      executionWorkspaceId: null,
      issueId: null,
      heartbeatRunId: runId,
      provider: "fake-plugin-reload",
      providerLeaseId: "plugin-lease-reload",
      metadata: { provider: "fake-plugin-reload", driver: "sandbox", reuseLease: false },
      failureReason: "acquire_rejected_teardown_failed",
    });
    const lease = (await environmentService(db).getLeaseById(orphan.id))!;

    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    // A not-ready plugin is the transient reload window, so the probe reports
    // not ready. The sweep skips the lease without a claim, so no attempt burns.
    await expect(
      runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }),
    ).resolves.toBe(false);
    expect((workerManager.call as ReturnType<typeof vi.fn>)).not.toHaveBeenCalled();

    // The plugin becomes ready, so the probe now reports ready and the sweep
    // proceeds.
    await db.update(plugins).set({ status: "ready" }).where(eq(plugins.id, pluginId));
    await expect(
      runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }),
    ).resolves.toBe(true);
  });

  it("reports the plugin cleanup provider not ready while the plugin is missing, then ready after it returns", async () => {
    // A pending_cleanup orphan targets a plugin-backed provider whose plugin row
    // is gone. A plugin reinstall removes and re-adds the plugin row, so the row
    // is missing for a short window. The sweep must not consume a finite retry
    // attempt in that window. So the readiness probe reports "not ready" while no
    // plugin row resolves the provider, and "ready" after the plugin returns.
    const pluginId = randomUUID();
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Cleanup Plugin Missing",
      config: { provider: "fake-plugin-missing", image: "fake:test", reuseLease: false },
    });

    // The worker manager reports the plugin worker as running, so the probe
    // result depends on the missing plugin row alone, not on a down worker.
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async () => {
        throw new Error("call must not run during a readiness probe");
      }),
    } as unknown as PluginWorkerManager;

    const orphan = await environmentService(db).insertPendingCleanupLease({
      companyId,
      environmentId: environment.id,
      executionWorkspaceId: null,
      issueId: null,
      heartbeatRunId: runId,
      provider: "fake-plugin-missing",
      providerLeaseId: "plugin-lease-missing",
      metadata: { provider: "fake-plugin-missing", driver: "sandbox", reuseLease: false },
      failureReason: "acquire_rejected_teardown_failed",
    });
    const lease = (await environmentService(db).getLeaseById(orphan.id))!;

    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    // No plugin row resolves the provider, so the probe reports not ready. The
    // sweep skips the lease without a claim, so no attempt burns while the plugin
    // reinstall is in flight.
    await expect(
      runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }),
    ).resolves.toBe(false);
    expect((workerManager.call as ReturnType<typeof vi.fn>)).not.toHaveBeenCalled();

    // The plugin reinstall completes, so a ready plugin row now resolves the
    // provider and the probe reports ready.
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "paperclip.fake-plugin-missing-sandbox-provider",
      packageName: "@paperclipai/plugin-fake-missing-sandbox",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "paperclip.fake-plugin-missing-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Fake Plugin Missing Sandbox Provider",
        description: "Test fake plugin provider missing readiness",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin-missing",
            kind: "sandbox_provider",
            displayName: "Fake Plugin Missing",
            configSchema: { type: "object" },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);
    await expect(
      runtimeWithPlugin.isPendingCleanupWorkerReady({ environment, lease }),
    ).resolves.toBe(true);
  });

  it("records an orphan with a null reference when a delete already removed the environment", async () => {
    // Ordering: a delete removes the environment before the acquire records the
    // orphan. The environment foreign key must not fail the insert. The record
    // persists with a null reference and its immutable provider metadata, so a
    // later sweep still tears the orphan down.
    const { companyId, environment } = await seedEnvironment({
      driver: "sandbox",
      name: "Delete Before Orphan Record",
      config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
    });

    // Remove the environment row first, so the record happens after the delete.
    await environmentService(db).remove(environment.id);

    const orphan = await environmentService(db).insertPendingCleanupLease({
      companyId,
      environmentId: environment.id,
      provider: "fake",
      providerLeaseId: "sandbox://fake/delete-before-record",
      metadata: { provider: "fake", image: "ubuntu:24.04", driver: "sandbox" },
      failureReason: "acquire_rejected_teardown_failed",
    });
    // The insert kept no environment reference, so the delete did not fail it.
    expect(orphan.environmentId).toBeNull();
    expect(orphan.status).toBe("pending_cleanup");

    const destroySpy = vi
      .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
      .mockResolvedValue(undefined);
    try {
      // The recorded metadata drives the teardown, so a null environment tears
      // the orphan down.
      const lease = await environmentService(db).getLeaseById(orphan.id);
      await runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! });
      expect(destroySpy).toHaveBeenCalledWith(
        expect.objectContaining({ providerLeaseId: "sandbox://fake/delete-before-record" }),
      );
    } finally {
      destroySpy.mockRestore();
    }
  });

  it("keeps the orphan and tears it down after the environment is deleted", async () => {
    // Ordering: an orphan is recorded, then the environment is deleted. The
    // foreign key sets the reference null instead of cascading the row away, so
    // the orphan survives. The sweep tears it down from the recorded metadata.
    const { companyId, environment } = await seedEnvironment({
      driver: "sandbox",
      name: "Delete After Orphan Record",
      config: { provider: "fake", image: "ubuntu:24.04", reuseLease: false },
    });

    const orphan = await environmentService(db).insertPendingCleanupLease({
      companyId,
      environmentId: environment.id,
      provider: "fake",
      providerLeaseId: "sandbox://fake/delete-after-record",
      metadata: { provider: "fake", image: "ubuntu:24.04", driver: "sandbox" },
      failureReason: "acquire_rejected_teardown_failed",
    });
    expect(orphan.environmentId).toBe(environment.id);

    // Delete the environment row directly. The foreign key sets the lease
    // reference null, so the orphan survives instead of being cascaded away.
    await environmentService(db).remove(environment.id);
    const survived = await db
      .select()
      .from(environmentLeases)
      .where(eq(environmentLeases.id, orphan.id))
      .then((r) => r[0]!);
    expect(survived.environmentId).toBeNull();
    expect(survived.status).toBe("pending_cleanup");

    const destroySpy = vi
      .spyOn(sandboxProviderRuntime, "destroySandboxProviderLease")
      .mockResolvedValue(undefined);
    try {
      // The recorded metadata drives the teardown, so a null environment tears
      // the orphan down after the delete set the reference null.
      const lease = await environmentService(db).getLeaseById(orphan.id);
      await runtime.retryPendingSandboxTeardown({ environment: null, lease: lease! });
      expect(destroySpy).toHaveBeenCalledWith(
        expect.objectContaining({ providerLeaseId: "sandbox://fake/delete-after-record" }),
      );
    } finally {
      destroySpy.mockRestore();
    }
  });

  // A plugin provider whose config declares an `apiKey` secret-ref field. The
  // orphan teardown must resolve that recorded ref even when the environment
  // binding is gone, so these tests register a provider with a real secret-ref
  // field, unlike the plain `fake-plugin` tests above.
  const SECRET_REF_PLUGIN_KEY = "paperclip.secret-plugin-sandbox-provider";
  const SECRET_REF_PROVIDER = "secret-plugin";

  async function registerSecretRefPluginProvider(): Promise<string> {
    const pluginId = randomUUID();
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: SECRET_REF_PLUGIN_KEY,
      packageName: "@paperclipai/plugin-secret-sandbox",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: SECRET_REF_PLUGIN_KEY,
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Secret Plugin Sandbox Provider",
        description: "Test plugin provider with a secret-ref config field",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: SECRET_REF_PROVIDER,
            kind: "sandbox_provider",
            displayName: "Secret Plugin",
            configSchema: {
              type: "object",
              properties: {
                apiKey: { type: "string", format: "secret-ref" },
              },
            },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);
    return pluginId;
  }

  // A worker manager mock that records the config of each destroy call, so a
  // test can assert the resolved API key reached the provider teardown.
  function createDestroyRecordingWorkerManager(pluginId: string): {
    workerManager: PluginWorkerManager;
    destroyConfigs: Array<Record<string, unknown> | undefined>;
  } {
    const destroyConfigs: Array<Record<string, unknown> | undefined> = [];
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string, args: any) => {
        if (method === "environmentDestroyLease") {
          destroyConfigs.push(args?.config as Record<string, unknown> | undefined);
          return {};
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
    } as unknown as PluginWorkerManager;
    return { workerManager, destroyConfigs };
  }

  async function createApiKeySecret(companyId: string, value: string): Promise<string> {
    const secret = await secretService(db).create(companyId, {
      name: `sandbox-cleanup-api-key-${randomUUID()}`,
      provider: "local_encrypted",
      value,
    });
    return secret.id;
  }

  it("tears down a secret-backed plugin orphan when a delete already removed the environment", async () => {
    // Ordering: a delete removed the environment (and its secret binding) before
    // the acquire recorded the orphan. The recorded config keeps the API-key
    // secret ref, so the teardown must resolve it without the environment
    // binding and send the resolved value to the provider destroy call.
    const pluginId = await registerSecretRefPluginProvider();
    const { companyId, environment } = await seedEnvironment({
      driver: "sandbox",
      name: "Secret Plugin Delete Before Record",
      config: { provider: SECRET_REF_PROVIDER, reuseLease: false },
    });
    const secretId = await createApiKeySecret(companyId, "old-provider-api-key");
    await secretService(db).createBinding({
      companyId,
      secretId,
      targetType: "environment",
      targetId: environment.id,
      configPath: "apiKey",
    });

    // Remove the environment first, so the record happens after the delete.
    await environmentService(db).remove(environment.id);

    const orphan = await environmentService(db).insertPendingCleanupLease({
      companyId,
      environmentId: environment.id,
      provider: SECRET_REF_PROVIDER,
      providerLeaseId: "secret-plugin-lease-delete-before",
      metadata: {
        provider: SECRET_REF_PROVIDER,
        driver: "sandbox",
        sandboxProviderPlugin: true,
        pluginId,
        pluginKey: SECRET_REF_PLUGIN_KEY,
        apiKey: secretId,
        reuseLease: false,
      },
      failureReason: "acquire_rejected_teardown_failed",
    });
    expect(orphan.environmentId).toBeNull();
    expect(orphan.status).toBe("pending_cleanup");

    const { workerManager, destroyConfigs } = createDestroyRecordingWorkerManager(pluginId);
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const lease = await environmentService(db).getLeaseById(orphan.id);
    vi.mocked(workerManager.call).mockImplementationOnce(async (_pluginId, _method, args: any) => {
      destroyConfigs.push(args.config);
      return { providerLeaseId: args.providerLeaseId, state: "destroyed" };
    });
    await expect(runtimeWithPlugin.retryPendingSandboxTeardown({ environment: null, lease: lease! }))
      .resolves.toEqual({ providerLeaseId: orphan.providerLeaseId, state: "destroyed" });
    expect(destroyConfigs).toHaveLength(1);
    // The recorded secret ref resolved to the old credential, and the resolved
    // value reached the provider teardown instead of the secret ref.
    expect(destroyConfigs[0]).toMatchObject({ apiKey: "old-provider-api-key" });
    expect(destroyConfigs[0]?.apiKey).not.toBe(secretId);
  });

  it("tears down a secret-backed plugin orphan after the environment is deleted", async () => {
    // Ordering: the orphan is recorded, then a delete removes the environment.
    // The foreign key sets the lease reference null, so the orphan survives. The
    // teardown resolves the recorded API-key ref without the environment.
    const pluginId = await registerSecretRefPluginProvider();
    const { companyId, environment } = await seedEnvironment({
      driver: "sandbox",
      name: "Secret Plugin Delete After Record",
      config: { provider: SECRET_REF_PROVIDER, reuseLease: false },
    });
    const secretId = await createApiKeySecret(companyId, "old-provider-api-key");
    await secretService(db).createBinding({
      companyId,
      secretId,
      targetType: "environment",
      targetId: environment.id,
      configPath: "apiKey",
    });

    const orphan = await environmentService(db).insertPendingCleanupLease({
      companyId,
      environmentId: environment.id,
      provider: SECRET_REF_PROVIDER,
      providerLeaseId: "secret-plugin-lease-delete-after",
      metadata: {
        provider: SECRET_REF_PROVIDER,
        driver: "sandbox",
        sandboxProviderPlugin: true,
        pluginId,
        pluginKey: SECRET_REF_PLUGIN_KEY,
        apiKey: secretId,
        reuseLease: false,
      },
      failureReason: "acquire_rejected_teardown_failed",
    });
    expect(orphan.environmentId).toBe(environment.id);

    // Delete the environment. The foreign key sets the lease reference null, so
    // the orphan survives and the current binding is gone.
    await environmentService(db).remove(environment.id);
    const survived = await db
      .select()
      .from(environmentLeases)
      .where(eq(environmentLeases.id, orphan.id))
      .then((r) => r[0]!);
    expect(survived.environmentId).toBeNull();

    const { workerManager, destroyConfigs } = createDestroyRecordingWorkerManager(pluginId);
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const lease = await environmentService(db).getLeaseById(orphan.id);
    await runtimeWithPlugin.retryPendingSandboxTeardown({ environment: null, lease: lease! });
    expect(destroyConfigs).toHaveLength(1);
    expect(destroyConfigs[0]).toMatchObject({ apiKey: "old-provider-api-key" });
  });

  it("tears down a secret-backed plugin orphan with the recorded credential after the provider is reconfigured", async () => {
    // Ordering: the orphan is recorded for the old credential, then the
    // environment is reconfigured to a new credential. The environment binding
    // now points to a different secret at the same path. The teardown must
    // resolve the OLD recorded ref, not the new binding, so it destroys the
    // sandbox the old credential provisioned.
    const pluginId = await registerSecretRefPluginProvider();
    const { companyId, environment } = await seedEnvironment({
      driver: "sandbox",
      name: "Secret Plugin Reconfigured",
      config: { provider: SECRET_REF_PROVIDER, reuseLease: false },
    });
    const oldSecretId = await createApiKeySecret(companyId, "old-provider-api-key");

    const orphan = await environmentService(db).insertPendingCleanupLease({
      companyId,
      environmentId: environment.id,
      provider: SECRET_REF_PROVIDER,
      providerLeaseId: "secret-plugin-lease-reconfigured",
      metadata: {
        provider: SECRET_REF_PROVIDER,
        driver: "sandbox",
        sandboxProviderPlugin: true,
        pluginId,
        pluginKey: SECRET_REF_PLUGIN_KEY,
        apiKey: oldSecretId,
        reuseLease: false,
      },
      failureReason: "acquire_rejected_teardown_failed",
    });
    expect(orphan.environmentId).toBe(environment.id);

    // Reconfigure the environment: a new secret replaces the binding at the same
    // path. The old recorded ref is no longer bound to the environment.
    const newSecretId = await createApiKeySecret(companyId, "new-provider-api-key");
    await secretService(db).createBinding({
      companyId,
      secretId: newSecretId,
      targetType: "environment",
      targetId: environment.id,
      configPath: "apiKey",
    });
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      config: { provider: SECRET_REF_PROVIDER, apiKey: newSecretId, reuseLease: false },
    });

    const { workerManager, destroyConfigs } = createDestroyRecordingWorkerManager(pluginId);
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const lease = await environmentService(db).getLeaseById(orphan.id);
    await runtimeWithPlugin.retryPendingSandboxTeardown({ environment: null, lease: lease! });
    expect(destroyConfigs).toHaveLength(1);
    // The teardown resolved the OLD recorded credential, not the new binding.
    expect(destroyConfigs[0]).toMatchObject({ apiKey: "old-provider-api-key" });
    expect(destroyConfigs[0]?.apiKey).not.toBe("new-provider-api-key");

    await environmentService(db).update(environment.id, { driver: "local", config: {} });
  });

  it("tears the recorded plugin provider lease down for a pending-cleanup orphan", async () => {
    const pluginId = randomUUID();
    const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
    const pluginConfig = { provider: "fake-plugin", image: "fake:test", reuseLease: false };
    const environment = {
      ...baseEnvironment,
      name: "Cleanup Retry Plugin",
      driver: "sandbox",
      config: pluginConfig,
    };
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: pluginConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "paperclip.fake-plugin-sandbox-provider",
      packageName: "@paperclipai/plugin-fake-sandbox",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "paperclip.fake-plugin-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Fake Plugin Sandbox Provider",
        description: "Test fake plugin provider",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            configSchema: { type: "object" },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);
    const otherCompanyId = randomUUID();
    await db.insert(companies).values({
      id: otherCompanyId,
      name: "Other Co Retry Plugin",
      issuePrefix: "OTG",
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(builtInManagedResources).values({
      companyId: otherCompanyId,
      bundleKey: "managed-environment",
      resourceKind: "environment",
      resourceKey: "managed-sandbox",
      resourceId: environment.id,
      stockVersion: "1",
      stockHash: "hash",
    });

    // The first destroy fails, so the acquire records a pending-cleanup orphan.
    // The sweep retries the destroy, which succeeds this time.
    let destroyAttempts = 0;
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentDestroyLease"] })),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "plugin-lease-3",
            metadata: { provider: "fake-plugin", image: "fake:test", reuseLease: false },
          };
        }
        if (method === "environmentDestroyLease") {
          destroyAttempts += 1;
          if (destroyAttempts === 1) {
            throw new Error("destroy failed");
          }
          return {};
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    await expect(
      runtimeWithPlugin.acquireRunLease({
        companyId,
        environment,
        issueId: null,
        heartbeatRunId: runId,
        persistedExecutionWorkspace: null,
        assertCompanyBinding: true,
      }),
    ).rejects.toMatchObject({ status: 403, details: { code: "environment_company_mismatch" } });

    const orphan = await db
      .select()
      .from(environmentLeases)
      .where(eq(environmentLeases.environmentId, environment.id))
      .then((r) => r[0]!);
    expect(orphan.status).toBe("pending_cleanup");
    expect(orphan.providerLeaseId).toBe("plugin-lease-3");

    const lease = await environmentService(db).getLeaseById(orphan.id);
    await runtimeWithPlugin.retryPendingSandboxTeardown({ environment: null, lease: lease! });
    expect(destroyAttempts).toBe(2);
    const destroyCall = (workerManager.call as unknown as ReturnType<typeof vi.fn>).mock.calls
      .filter((callArgs) => callArgs[1] === "environmentDestroyLease")
      .at(-1);
    expect(destroyCall?.[2]).toMatchObject({ providerLeaseId: "plugin-lease-3" });

    await environmentService(db).update(environment.id, { driver: "local", config: {} });
  });

  it("uses plugin-backed sandbox config for execute and release", async () => {
    const pluginId = randomUUID();
    const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
    const fakePluginConfig = {
      provider: "fake-plugin",
      image: "fake:test",
      timeoutMs: 1234,
      reuseLease: false,
    };
    const environment = {
      ...baseEnvironment,
      name: "Fake Plugin Sandbox",
      driver: "sandbox",
      config: fakePluginConfig,
    };
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: fakePluginConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "paperclip.fake-plugin-sandbox-provider",
      packageName: "@paperclipai/plugin-fake-sandbox",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "paperclip.fake-plugin-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Fake Plugin Sandbox Provider",
        description: "Test fake plugin provider",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            configSchema: { type: "object" },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string, params: any) => {
        expect(params.config).toEqual(expect.objectContaining({
          image: "fake:test",
          timeoutMs: 1234,
          reuseLease: false,
        }));
        expect(params.config).not.toHaveProperty("provider");
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "sandbox-1",
            metadata: {
              provider: "fake-plugin",
              image: "fake:test",
              timeoutMs: 1234,
              reuseLease: false,
              remoteCwd: "/workspace",
            },
          };
        }
        if (method === "environmentExecute") {
          return {
            exitCode: 0,
            signal: null,
            timedOut: false,
            stdout: "ok\n",
            stderr: "",
          };
        }
        if (method === "environmentReleaseLease") {
          expect(params.config).toEqual({
            image: "fake:test",
            timeoutMs: 1234,
            reuseLease: false,
          });
          expect(params.config).not.toHaveProperty("driver");
          expect(params.config).not.toHaveProperty("executionWorkspaceMode");
          expect(params.config).not.toHaveProperty("pluginId");
          expect(params.config).not.toHaveProperty("pluginKey");
          expect(params.config).not.toHaveProperty("providerMetadata");
          expect(params.config).not.toHaveProperty("provider");
          expect(params.config).not.toHaveProperty("sandboxProviderPlugin");
          return undefined;
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const acquired = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
    });
    const executed = await runtimeWithPlugin.execute({
      environment,
      lease: acquired.lease,
      command: "printf",
      args: ["ok"],
      cwd: "/workspace",
      env: {},
      timeoutMs: 1000,
    });

    await environmentService(db).update(environment.id, {
      driver: "local",
      config: {},
    });
    const released = await runtimeWithPlugin.releaseRunLeases(runId);

    expect(executed.stdout).toBe("ok\n");
    expect(released).toHaveLength(1);
    expect(released[0]?.lease.status).toBe("released");
    // The execute call carries the optional log sink as the fifth argument; it
    // is undefined when the caller passes no sink.
    expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentExecute", expect.anything(), 31000, undefined);
    expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentReleaseLease", expect.anything(), 31234);
  });

  // Build the fake plugin fixture for the run-parent release tests below. A
  // plugin sandbox provider can open a persistent session on the first command
  // and delete it on lease release; the delete emits a provider
  // `session.close` span. The host mints that span's parent from the active
  // step context at the release RPC. So the release must run under the run
  // parent, or the span loses its traceparent and the backend drops it.
  async function seedFakePluginSandbox() {
    const pluginId = randomUUID();
    const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
    const fakePluginConfig = {
      provider: "fake-plugin",
      image: "fake:test",
      timeoutMs: 1234,
      reuseLease: false,
    };
    const environment = {
      ...baseEnvironment,
      name: "Fake Plugin Sandbox",
      driver: "sandbox",
      config: fakePluginConfig,
    };
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: fakePluginConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "paperclip.fake-plugin-sandbox-provider",
      packageName: "@paperclipai/plugin-fake-sandbox",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "paperclip.fake-plugin-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Fake Plugin Sandbox Provider",
        description: "Test fake plugin provider",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            configSchema: { type: "object" },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);
    return { companyId, environment, runId, pluginId };
  }

  it("runs a plugin-backed lease release under the run-time exec parent, so the teardown span keeps a valid traceparent", async () => {
    const { companyId, environment, runId, pluginId } = await seedFakePluginSandbox();

    // The active step context observed at the moment the host issues the release
    // RPC. The real worker manager mints the provider-span traceparent from
    // exactly this value, so a non-null context with a valid parent proves the
    // teardown span keeps a host-minted parent.
    let releaseStepContext: ReturnType<typeof getActiveStepContext> = null;
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "sandbox-1",
            metadata: {
              provider: "fake-plugin",
              image: "fake:test",
              timeoutMs: 1234,
              reuseLease: false,
              remoteCwd: "/workspace",
            },
          };
        }
        if (method === "environmentExecute") {
          return { exitCode: 0, signal: null, timedOut: false, stdout: "ok\n", stderr: "" };
        }
        if (method === "environmentReleaseLease") {
          releaseStepContext = getActiveStepContext();
          return undefined;
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const acquired = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
    });

    // Build a real host run-parent context and run the first command under it,
    // exactly as the run drives an exec. The driver records this context for the
    // lease and replays it around the later release RPC.
    const runParent: StartupSpanContext = trace.setSpanContext(ROOT_CONTEXT, {
      traceId: "0af7651916cd43dd8448eb211c80319c",
      spanId: "b7ad6b7169203331",
      traceFlags: 1,
      isRemote: false,
    });
    await runWithRuntimeParent(runParent, () =>
      runtimeWithPlugin.execute({
        environment,
        lease: acquired.lease,
        command: "printf",
        args: ["ok"],
        cwd: "/workspace",
        env: {},
        timeoutMs: 1000,
      }),
    );

    await environmentService(db).update(environment.id, { driver: "local", config: {} });
    const released = await runtimeWithPlugin.releaseRunLeases(runId);

    expect(released).toHaveLength(1);
    // The release RPC ran under the recorded run parent, so the host mints a
    // valid W3C traceparent from it. A dropped span would show a null context.
    expect(releaseStepContext).not.toBeNull();
    expect(releaseStepContext?.parentContext).toBe(runParent);
    expect(traceparentFromContextToken(releaseStepContext?.parentContext)).toBe(
      "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",
    );
  });

  it("releases a plugin-backed lease with no active step context when the run drove no exec", async () => {
    const { companyId, environment, runId, pluginId } = await seedFakePluginSandbox();

    let releaseStepContext: ReturnType<typeof getActiveStepContext> = null;
    let releaseCalled = false;
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "sandbox-1",
            metadata: {
              provider: "fake-plugin",
              image: "fake:test",
              timeoutMs: 1234,
              reuseLease: false,
              remoteCwd: "/workspace",
            },
          };
        }
        if (method === "environmentReleaseLease") {
          releaseCalled = true;
          releaseStepContext = getActiveStepContext();
          return undefined;
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
    });
    await environmentService(db).update(environment.id, { driver: "local", config: {} });
    const released = await runtimeWithPlugin.releaseRunLeases(runId);

    expect(released).toHaveLength(1);
    // No command ran, so no session opened and no teardown span is emitted. The
    // release must not invent a parent from an unrelated ambient context; it
    // runs unwrapped, exactly as before the fix.
    expect(releaseCalled).toBe(true);
    expect(releaseStepContext).toBeNull();
  });

  it("forwards the bypassSession flag to the plugin execute RPC so a pre-run command skips the session", async () => {
    const { companyId, environment, runId, pluginId } = await seedFakePluginSandbox();

    // Capture the params of each environmentExecute RPC, so the test can assert
    // the host forwards `bypassSession` to the provider unchanged.
    const executeParams: Array<Record<string, unknown>> = [];
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string, params: Record<string, unknown>) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "sandbox-1",
            metadata: {
              provider: "fake-plugin",
              image: "fake:test",
              timeoutMs: 1234,
              reuseLease: false,
              remoteCwd: "/workspace",
            },
          };
        }
        if (method === "environmentExecute") {
          executeParams.push(params);
          return { exitCode: 0, signal: null, timedOut: false, stdout: "ok\n", stderr: "" };
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const acquired = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
    });

    // A pre-run command (the provision command) sets bypassSession explicitly.
    await runtimeWithPlugin.execute({
      environment,
      lease: acquired.lease,
      command: "bash",
      args: ["-lc", "true"],
      cwd: "/workspace",
      env: {},
      timeoutMs: 1000,
      bypassSession: true,
    });
    // An in-run command runs under the run-parent context, so the host does not
    // bypass the session and the provider opens/uses the session.
    const runParent: StartupSpanContext = trace.setSpanContext(ROOT_CONTEXT, {
      traceId: "0af7651916cd43dd8448eb211c80319c",
      spanId: "b7ad6b7169203331",
      traceFlags: 1,
      isRemote: false,
    });
    await runWithRuntimeParent(runParent, () =>
      runtimeWithPlugin.execute({
        environment,
        lease: acquired.lease,
        command: "printf",
        args: ["ok"],
        cwd: "/workspace",
        env: {},
        timeoutMs: 1000,
      }),
    );

    expect(executeParams).toHaveLength(2);
    expect(executeParams[0]?.bypassSession).toBe(true);
    // An in-run command carries a run parent, so it does not bypass the session.
    expect(executeParams[1]?.bypassSession).toBe(false);
  });

  it("bypasses the session for a context-less command so the setup span keeps a run parent", async () => {
    const { companyId, environment, runId, pluginId } = await seedFakePluginSandbox();

    // Capture the params of each environmentExecute RPC, so the test can assert
    // the host derives `bypassSession` from the active run-parent context.
    const executeParams: Array<Record<string, unknown>> = [];
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string, params: Record<string, unknown>) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "sandbox-1",
            metadata: {
              provider: "fake-plugin",
              image: "fake:test",
              timeoutMs: 1234,
              reuseLease: false,
              remoteCwd: "/workspace",
            },
          };
        }
        if (method === "environmentExecute") {
          executeParams.push(params);
          return { exitCode: 0, signal: null, timedOut: false, stdout: "ok\n", stderr: "" };
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const acquired = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
    });

    // A command with no active run-parent context (a pre-run install/probe or
    // the agent process launch) runs before the run trace is active. The host
    // mints no plugin RPC traceparent for it, so a session opened here would drop
    // its setup span. The host bypasses the session for such a command.
    await runtimeWithPlugin.execute({
      environment,
      lease: acquired.lease,
      command: "sh",
      args: ["-c", "command -v claude"],
      cwd: "/workspace",
      env: {},
      timeoutMs: 1000,
    });

    expect(executeParams).toHaveLength(1);
    expect(executeParams[0]?.bypassSession).toBe(true);
  });

  it("builds the workspace-realization record with referenced sources for a plugin-backed sandbox realize", async () => {
    // A provider plugin realize handler returns only its realized cwd and provider metadata; it does
    // not build the workspace-realization record. The server must build that record from the run
    // request, so the referenced (mentioned) project sources reach the adapter through
    // `realization.additional`. Without the record the sandbox agent never receives the mentioned
    // projects. This test drives the plugin-backed sandbox realize path and asserts the referenced
    // source survives into the returned record.
    const pluginId = randomUUID();
    const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
    const fakePluginConfig = {
      provider: "fake-plugin",
      image: "fake:test",
      timeoutMs: 1234,
      reuseLease: false,
    };
    const environment = {
      ...baseEnvironment,
      name: "Fake Plugin Sandbox Realize",
      driver: "sandbox",
      config: fakePluginConfig,
    };
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: fakePluginConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "paperclip.fake-plugin-sandbox-provider",
      packageName: "@paperclipai/plugin-fake-sandbox",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "paperclip.fake-plugin-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Fake Plugin Sandbox Provider",
        description: "Test fake plugin provider",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            configSchema: { type: "object" },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "sandbox-realize-1",
            metadata: {
              provider: "fake-plugin",
              image: "fake:test",
              timeoutMs: 1234,
              reuseLease: false,
              remoteCwd: "/workspace",
            },
          };
        }
        if (method === "environmentRealizeWorkspace") {
          // Mimic a real provider (for example Daytona): return only the realized cwd and provider
          // metadata, never a `workspaceRealization` record.
          return {
            cwd: "/workspace/project",
            metadata: {
              provider: "fake-plugin",
              remoteCwd: "/workspace/project",
            },
          };
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const acquired = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
    });
    const workspaceRealizationRequest = {
      version: 1,
      adapterType: "codex_local",
      companyId,
      environmentId: environment.id,
      executionWorkspaceId: null,
      issueId: null,
      heartbeatRunId: runId,
      requestedMode: "ephemeral",
      source: {
        kind: "project_primary",
        localPath: "/tmp/anchor",
        projectId: "anchor-project",
        projectWorkspaceId: "anchor-workspace",
        repoUrl: null,
        repoRef: null,
        strategy: "project_primary",
        branchName: null,
        worktreePath: null,
      },
      additionalSources: [
        {
          localPath: "/tmp/referenced-project",
          projectId: "referenced-project-1",
          projectWorkspaceId: "referenced-workspace-1",
          repoUrl: null,
          repoRef: null,
        },
      ],
    };
    const realized = await runtimeWithPlugin.realizeWorkspace({
      environment,
      lease: acquired.lease,
      workspace: {
        localPath: "/tmp/anchor",
        mode: "ephemeral",
        metadata: { workspaceRealizationRequest },
      },
    });

    // The provider realized cwd and provider metadata survive.
    expect(realized.cwd).toBe("/workspace/project");
    expect(realized.metadata?.provider).toBe("fake-plugin");
    // The server-built record carries the referenced source through `additional`, so the adapter can
    // stage the mentioned project into the sandbox.
    const realization = realized.metadata?.workspaceRealization as Record<string, unknown> | undefined;
    expect(realization).toBeDefined();
    expect(realization?.additional).toEqual([
      expect.objectContaining({
        path: "/tmp/referenced-project",
        projectId: "referenced-project-1",
      }),
    ]);
  });

  it("builds the workspace-realization record with referenced sources for a built-in sandbox realize", async () => {
    // The sandbox driver `realizeWorkspace` has two exits: a plugin-backed provider and a
    // built-in provider. Both must build the same workspace-realization record, so the
    // referenced (mentioned) project sources reach the adapter through `realization.additional`.
    // The test above covers the plugin exit. This test covers the built-in exit (no provider
    // plugin), so a regression on either exit fails a test.
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Fake Sandbox Realize",
      config: {
        provider: "fake",
        image: "ubuntu:24.04",
        reuseLease: true,
      },
    });

    const acquired = await runtime.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
    });

    const workspaceRealizationRequest = {
      version: 1,
      adapterType: "codex_local",
      companyId,
      environmentId: environment.id,
      executionWorkspaceId: null,
      issueId: null,
      heartbeatRunId: runId,
      requestedMode: "ephemeral",
      source: {
        kind: "project_primary",
        localPath: "/tmp/anchor",
        projectId: "anchor-project",
        projectWorkspaceId: "anchor-workspace",
        repoUrl: null,
        repoRef: null,
        strategy: "project_primary",
        branchName: null,
        worktreePath: null,
      },
      additionalSources: [
        {
          localPath: "/tmp/referenced-project",
          projectId: "referenced-project-1",
          projectWorkspaceId: "referenced-workspace-1",
          repoUrl: null,
          repoRef: null,
        },
      ],
    };
    const realized = await runtime.realizeWorkspace({
      environment,
      lease: acquired.lease,
      workspace: {
        localPath: "/tmp/anchor",
        mode: "ephemeral",
        metadata: { workspaceRealizationRequest },
      },
    });

    // The built-in exit builds the record and carries the referenced source through `additional`,
    // so the adapter can stage the mentioned project into the sandbox.
    const realization = realized.metadata?.workspaceRealization as Record<string, unknown> | undefined;
    expect(realization).toBeDefined();
    expect(realization?.additional).toEqual([
      expect.objectContaining({
        path: "/tmp/referenced-project",
        projectId: "referenced-project-1",
      }),
    ]);
  });

  it("uses resolved secret-ref config for plugin-backed sandbox execute and release", async () => {
    const pluginId = randomUUID();
    const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
    const apiSecret = await secretService(db).create(companyId, {
      name: `secure-plugin-api-key-${randomUUID()}`,
      provider: "local_encrypted",
      value: "resolved-provider-key",
    });
    const providerConfig = {
      provider: "secure-plugin",
      template: "base",
      apiKey: apiSecret.id,
      timeoutMs: 1234,
      reuseLease: false,
    };
    const environment = {
      ...baseEnvironment,
      name: "Secure Plugin Sandbox",
      driver: "sandbox",
      config: providerConfig,
    };
    await secretService(db).createBinding({
      companyId,
      secretId: apiSecret.id,
      targetType: "environment",
      targetId: environment.id,
      configPath: "apiKey",
    });
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: providerConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "acme.secure-sandbox-provider",
      packageName: "@acme/secure-sandbox-provider",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.secure-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Secure Sandbox Provider",
        description: "Test schema-driven provider",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "secure-plugin",
            kind: "sandbox_provider",
            displayName: "Secure Sandbox",
            configSchema: {
              type: "object",
              properties: {
                template: { type: "string" },
                apiKey: { type: "string", format: "secret-ref" },
                timeoutMs: { type: "number" },
                reuseLease: { type: "boolean" },
              },
            },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string, params: any) => {
        expect(params.config.apiKey).toBe("resolved-provider-key");
        expect(params.config).not.toHaveProperty("provider");
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "sandbox-1",
            metadata: {
              provider: "secure-plugin",
              template: "base",
              apiKey: "resolved-provider-key",
              timeoutMs: 1234,
              reuseLease: false,
              sandboxId: "sandbox-1",
              remoteCwd: "/workspace",
            },
          };
        }
        if (method === "environmentExecute") {
          return {
            exitCode: 0,
            signal: null,
            timedOut: false,
            stdout: "ok\n",
            stderr: "",
          };
        }
        if (method === "environmentReleaseLease") {
          return undefined;
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const acquired = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
    });
    expect(acquired.lease.metadata).toMatchObject({
      provider: "secure-plugin",
      template: "base",
      apiKey: apiSecret.id,
      timeoutMs: 1234,
      sandboxId: "sandbox-1",
    });
    const executed = await runtimeWithPlugin.execute({
      environment,
      lease: acquired.lease,
      command: "printf",
      args: ["ok"],
      cwd: "/workspace",
      env: {},
    });

    await environmentService(db).update(environment.id, {
      driver: "local",
      config: {},
    });
    const released = await runtimeWithPlugin.releaseRunLeases(runId);

    expect(executed.stdout).toBe("ok\n");
    expect(released).toHaveLength(1);
    expect(released[0]?.lease.status).toBe("released");
    // The execute call carries the optional log sink as the fifth argument; it
    // is undefined when the caller passes no sink.
    expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentExecute", expect.objectContaining({
      config: expect.objectContaining({
        apiKey: "resolved-provider-key",
      }),
    }), 31234, undefined);
    expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentReleaseLease", expect.objectContaining({
      config: expect.objectContaining({
        apiKey: "resolved-provider-key",
      }),
    }), 31234);
  });

  it("resolves persisted secret_ref binding objects in sandbox provider config at runtime", async () => {
    const pluginId = randomUUID();
    const { companyId, environment: baseEnvironment } = await seedEnvironment();
    const apiSecret = await secretService(db).create(companyId, {
      name: `secure-plugin-api-key-${randomUUID()}`,
      provider: "local_encrypted",
      value: "resolved-provider-key",
    });
    const providerConfig = {
      provider: "secure-plugin",
      template: "base",
      apiKey: { type: "secret_ref", secretId: apiSecret.id, version: "latest" },
      timeoutMs: 1234,
      reuseLease: false,
    };
    const environment = {
      ...baseEnvironment,
      name: "Secure Plugin Sandbox",
      driver: "sandbox",
      config: providerConfig,
    };
    await secretService(db).createBinding({
      companyId,
      secretId: apiSecret.id,
      targetType: "environment",
      targetId: environment.id,
      configPath: "apiKey",
    });
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: providerConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "acme.secure-sandbox-provider",
      packageName: "@acme/secure-sandbox-provider",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.secure-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Secure Sandbox Provider",
        description: "Test schema-driven provider",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "secure-plugin",
            kind: "sandbox_provider",
            displayName: "Secure Sandbox",
            configSchema: {
              type: "object",
              properties: {
                template: { type: "string" },
                apiKey: { type: "string", format: "secret-ref" },
                timeoutMs: { type: "number" },
                reuseLease: { type: "boolean" },
              },
            },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);

    const resolved = await resolveEnvironmentDriverConfigForRuntime(db, companyId, environment);

    expect(resolved.driver).toBe("sandbox");
    expect(resolved.config).toMatchObject({
      provider: "secure-plugin",
      template: "base",
      apiKey: "resolved-provider-key",
    });
  });

  it("waits briefly for a ready sandbox provider plugin worker to come online", async () => {
    const pluginId = randomUUID();
    const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
    const providerConfig = {
      provider: "fake-plugin",
      image: "fake:test",
      timeoutMs: 1234,
      reuseLease: false,
    };
    const environment = {
      ...baseEnvironment,
      name: "Eventually Running Plugin Sandbox",
      driver: "sandbox",
      config: providerConfig,
    };
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: providerConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "acme.eventually-running-sandbox-provider",
      packageName: "@acme/eventually-running-sandbox-provider",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.eventually-running-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Eventually Running Sandbox Provider",
        description: "Test plugin worker startup grace period",
        author: "Acme",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            configSchema: { type: "object" },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);

    let runningChecks = 0;
    const workerManager = {
      isRunning: vi.fn((id: string) => {
        if (id !== pluginId) return false;
        runningChecks += 1;
        return runningChecks >= 3;
      }),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "sandbox-1",
            metadata: {
              provider: "fake-plugin",
              image: "fake:test",
              timeoutMs: 1234,
              reuseLease: false,
            },
          };
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, {
      pluginWorkerManager: workerManager,
      pluginWorkerReadyTimeoutMs: 25,
      pluginWorkerReadyPollMs: 1,
    });

    const acquired = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
    });

    expect(acquired.lease.providerLeaseId).toBe("sandbox-1");
    expect(workerManager.isRunning).toHaveBeenCalledTimes(3);
    expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234);
  });

  it("throws a worker-not-running error once the readiness deadline is exhausted", async () => {
    const pluginId = randomUUID();
    const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
    const providerConfig = {
      provider: "fake-plugin",
      image: "fake:test",
      timeoutMs: 1234,
      reuseLease: false,
    };
    const environment = {
      ...baseEnvironment,
      name: "Never Running Plugin Sandbox",
      driver: "sandbox",
      config: providerConfig,
    };
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: providerConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "acme.never-running-sandbox-provider",
      packageName: "@acme/never-running-sandbox-provider",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.never-running-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Never Running Sandbox Provider",
        description: "Test plugin worker that never comes online before the readiness deadline",
        author: "Acme",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            configSchema: { type: "object" },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);

    const workerManager = {
      isRunning: vi.fn(() => false),
      call: vi.fn(async (_pluginId: string, method: string) => {
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, {
      pluginWorkerManager: workerManager,
      pluginWorkerReadyTimeoutMs: 25,
      pluginWorkerReadyPollMs: 1,
    });

    await expect(
      runtimeWithPlugin.acquireRunLease({
        companyId,
        environment,
        issueId: null,
        heartbeatRunId: runId,
        persistedExecutionWorkspace: null,
      }),
    ).rejects.toThrow(/worker is not running/);
    // Confirms the loop actually polled repeatedly across the deadline window
    // instead of giving up after a single check.
    expect(workerManager.isRunning.mock.calls.length).toBeGreaterThanOrEqual(2);
  });

  it("keeps polling across a longer worker restart window and succeeds once the handle registers late", async () => {
    const pluginId = randomUUID();
    const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
    const providerConfig = {
      provider: "fake-plugin",
      image: "fake:test",
      timeoutMs: 1234,
      reuseLease: false,
    };
    const environment = {
      ...baseEnvironment,
      name: "Registered Late Plugin Sandbox",
      driver: "sandbox",
      config: providerConfig,
    };
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: providerConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "acme.registered-late-sandbox-provider",
      packageName: "@acme/registered-late-sandbox-provider",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.registered-late-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Registered Late Sandbox Provider",
        description: "Test plugin worker handle that stays unregistered/starting for most of the readiness window",
        author: "Acme",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            configSchema: { type: "object" },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);

    // Simulates a worker process restart: the handle is absent (or "starting")
    // for most of the readiness window and only reports running with several
    // checks left before the deadline. A larger attempt count than the
    // existing "waits briefly" sanity test, to prove the loop is bound by the
    // deadline rather than by a small fixed number of attempts.
    const readyAfterChecks = 8;
    let checks = 0;
    const workerManager = {
      isRunning: vi.fn((id: string) => {
        if (id !== pluginId) return false;
        checks += 1;
        return checks >= readyAfterChecks;
      }),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "sandbox-registered-late",
            metadata: {
              provider: "fake-plugin",
              image: "fake:test",
              timeoutMs: 1234,
              reuseLease: false,
            },
          };
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, {
      pluginWorkerManager: workerManager,
      pluginWorkerReadyTimeoutMs: 200,
      pluginWorkerReadyPollMs: 1,
    });

    const acquired = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
    });

    expect(acquired.lease.providerLeaseId).toBe("sandbox-registered-late");
    expect(checks).toBe(readyAfterChecks);
  });

  it.each([
    { label: "explicit provider and bridge budgets", config: { timeoutMs: 1_234, bridgeRequestTimeoutMs: 40_000 }, declared: 300_000, expected: 70_000 },
    { label: "declared acquisition default", config: {}, declared: 300_000, expected: 330_000 },
    { label: "explicit shorter provider budget", config: { timeoutMs: 5_000 }, declared: 300_000, expected: 35_000 },
    { label: "bridge budget does not shorten acquisition default", config: { bridgeRequestTimeoutMs: 40_000 }, declared: 300_000, expected: 330_000 },
    { label: "bridge extends the acquisition default", config: { bridgeRequestTimeoutMs: 400_000 }, declared: 300_000, expected: 430_000 },
    { label: "undeclared provider retains worker default", config: {}, declared: undefined, expected: undefined },
  ])("uses $label for plugin-backed sandbox acquisition", async ({ config, declared, expected }) => {
    const pluginId = randomUUID();
    const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
    const providerConfig = {
      provider: "fake-plugin",
      image: "fake:test",
      ...config,
      reuseLease: false,
    };
    const environment = {
      ...baseEnvironment,
      name: "Long Lease Plugin Sandbox",
      driver: "sandbox",
      config: providerConfig,
    };
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: providerConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "acme.long-lease-sandbox-provider",
      packageName: "@acme/long-lease-sandbox-provider",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.long-lease-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Long Lease Sandbox Provider",
        description: "Test plugin worker acquire timeout",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            defaultAcquireTimeoutMs: declared,
            // A timeoutMs schema default can mean lease lifetime (for example
            // E2B). Only the dedicated declaration sets the host RPC budget.
            configSchema: { type: "object", properties: { timeoutMs: { type: "number", default: 3_600_000 } } },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);

    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "sandbox-1",
            metadata: {
              provider: "fake-plugin",
              image: "fake:test",
              ...config,
              reuseLease: false,
            },
          };
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const acquired = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
    });

    expect(acquired.lease.providerLeaseId).toBe("sandbox-1");
    expect(workerManager.call).toHaveBeenCalledWith(
      pluginId,
      "environmentAcquireLease",
      expect.objectContaining({
        driverKey: "fake-plugin",
        config: {
          image: "fake:test",
          ...config,
          reuseLease: false,
        },
      }),
      expected,
    );
  });

  it("preserves the exact lease when plugin-backed sandbox resume throws", async () => {
    const pluginId = randomUUID();
    const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
    const providerConfig = {
      provider: "fake-plugin",
      image: "fake:test",
      timeoutMs: 1234,
      reuseLease: true,
    };
    const environment = {
      ...baseEnvironment,
      name: "Reusable Plugin Sandbox",
      driver: "sandbox",
      config: providerConfig,
    };
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: providerConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "acme.fake-sandbox-provider",
      packageName: "@acme/fake-sandbox-provider",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.fake-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Fake Sandbox Provider",
        description: "Test schema-driven provider",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            supportsReusableLeases: true,
            configSchema: {
              type: "object",
              properties: {
                image: { type: "string" },
                timeoutMs: { type: "number" },
                reuseLease: { type: "boolean" },
              },
            },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);
    const executionWorkspaceId = randomUUID();
    const projectId = randomUUID();
    await db.insert(projects).values({
      id: projectId,
      companyId,
      name: `Workspace ${projectId.slice(0, 8)}`,
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(executionWorkspaces).values({
      id: executionWorkspaceId,
      companyId,
      projectId,
      mode: "shared_workspace",
      strategyType: "project_primary",
      name: "Reusable workspace",
      status: "active",
      providerType: "local_fs",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    const staleLease = await environmentService(db).acquireLease({
      companyId,
      environmentId: environment.id,
      executionWorkspaceId,
      heartbeatRunId: runId,
      leasePolicy: "reuse_by_environment",
      provider: "fake-plugin",
      providerLeaseId: "stale-plugin-lease",
      metadata: {
        agentId,
        driver: "sandbox",
        pluginId,
        pluginKey: "acme.fake-sandbox-provider",
        sandboxProviderPlugin: true,
        provider: "fake-plugin",
        image: "fake:test",
        timeoutMs: 1234,
        reuseLease: true,
        reusableSandboxLease: {
          version: 1,
          companyId,
          environmentId: environment.id,
          executionWorkspaceId,
          agentId,
          adapterType: null,
          provider: "fake-plugin",
          runtimeFingerprint: reusableRuntimeFingerprint({
            provider: "fake-plugin",
            adapterType: null,
            config: providerConfig,
          }),
        },
      },
    });

    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentResumeLease") {
          throw new Error("stale sandbox");
        }
        if (method === "environmentDestroyLease") {
          return undefined;
        }
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "fresh-plugin-lease",
            metadata: {
              provider: "fake-plugin",
              image: "fake:test",
              timeoutMs: 1234,
              reuseLease: true,
              remoteCwd: "/workspace",
            },
          };
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    await expect(runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      agentId,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: {
        id: executionWorkspaceId,
        mode: "shared_workspace",
      },
    })).rejects.toThrow("the lease was preserved and no replacement was created");

    expect(workerManager.call).toHaveBeenNthCalledWith(1, pluginId, "environmentResumeLease", expect.objectContaining({
      driverKey: "fake-plugin",
      providerLeaseId: "stale-plugin-lease",
    }), 31234);
    expect(workerManager.call).not.toHaveBeenCalledWith(
      pluginId,
      "environmentDestroyLease",
      expect.anything(),
      expect.anything(),
    );
    expect(workerManager.call).not.toHaveBeenCalledWith(
      pluginId,
      "environmentAcquireLease",
      expect.anything(),
      expect.anything(),
    );
    await expect(environmentService(db).getLeaseById(staleLease.id)).resolves.toMatchObject({
      status: "active",
    });
  });

  it("resumes only the exact stopped sandbox lifecycle without acquiring or reseeding", async () => {
    const seeded = await seedReusablePluginSandboxLease("paperclip_runner");
    await environmentService(db).releaseLease(seeded.reusableLease.id, "released", { cleanupStatus: "success" });
    const lease = (await environmentService(db).getLeaseById(seeded.reusableLease.id))!;
    const call = vi.fn(async (_id: string, method: string) => {
      if (method !== "environmentResumeLease") throw new Error("Only exact resume is permitted");
      return { providerLeaseId: lease.providerLeaseId, metadata: { remoteCwd: "/workspace" } };
    });
    const workerManager = { isRunning: () => true, call,
      getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] }),
    } as unknown as PluginWorkerManager;
    const runtime = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
    await expect(runtime.resumeRunLease({ environment: seeded.environment, lease })).resolves.toMatchObject({ providerLeaseId: lease.providerLeaseId });
    expect(call).toHaveBeenCalledOnce();
    expect(call).toHaveBeenCalledWith(seeded.pluginId, "environmentResumeLease", expect.objectContaining({
      companyId: seeded.companyId, environmentId: seeded.environment.id, providerLeaseId: lease.providerLeaseId,
      leaseMetadata: lease.metadata,
    }), expect.any(Number));
    expect((await environmentService(db).getLeaseById(lease.id))?.status).toBe("released");
    expect(await db.select().from(environmentLeases)).toHaveLength(1);
  });

  it.each(["stopped", "unconfirmed", "foreign_marker", "competing_owner", "retained_owner", "no_stop_capability", "live_request", "late_receipt", "duplicate_key", "owner_stopped", "owner_driver_changed", "owner_missing", "legacy_intent", "foreign_plugin_pin", "v2_missing_pin"])("recovers a pending export resume with stop-only cleanup after restart: %s", async outcome => {
    const seeded = await seedReusablePluginSandboxLease("paperclip_runner");
    const lease = seeded.reusableLease;
    const requestId = randomUUID();
    if (["duplicate_key", "owner_stopped", "owner_driver_changed", "owner_missing", "legacy_intent"].includes(outcome)) {
      const [owner] = await db.select().from(plugins).where(eq(plugins.id, seeded.pluginId));
      await db.insert(plugins).values({ ...owner, id: randomUUID(), pluginKey: "other.same-driver", packageName: "@other/same-driver",
        installOrder: -1, manifestJson: { ...owner.manifestJson, id: "other.same-driver" } });
      if (outcome === "owner_driver_changed") await db.update(plugins).set({ manifestJson: { ...owner.manifestJson, environmentDrivers: [] } }).where(eq(plugins.id, seeded.pluginId));
      if (outcome === "owner_missing") await db.delete(plugins).where(eq(plugins.id, seeded.pluginId));
    }
    await db.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed", metadata: {
      ...lease.metadata,
      nativeWorkspaceExportResume: { schema: outcome === "v2_missing_pin" ? "paperclip.workspace-export-resume.v2" : "paperclip.workspace-export-resume.v1", requestId, companyId: seeded.companyId,
        ...(["legacy_intent", "v2_missing_pin"].includes(outcome) ? {} : { pluginId: outcome === "foreign_plugin_pin" ? randomUUID() : seeded.pluginId }),
        runId: outcome === "foreign_marker" ? randomUUID() : lease.heartbeatRunId, leaseId: lease.id,
        provider: lease.provider, providerLeaseId: lease.providerLeaseId, resultId: randomUUID() },
      pendingCleanupAttemptId: requestId, pendingCleanupInFlight: true,
      pendingCleanupLeaseExpiresAtMs: Date.now() + (outcome === "live_request" ? 60_000 : -1),
    } }).where(eq(environmentLeases.id, lease.id));
    if (["competing_owner", "retained_owner"].includes(outcome)) await db.insert(environmentLeases).values({ companyId: seeded.companyId,
      environmentId: seeded.environment.id, status: outcome === "retained_owner" ? "retained" : "active", provider: lease.provider, providerLeaseId: lease.providerLeaseId });
    const call = vi.fn(async (_id: string, method: string) => {
      if (method !== "environmentStopLease") throw new Error("Saved workspace must never be destroyed");
      if (outcome === "late_receipt") await db.update(environmentLeases).set({ status: "active", heartbeatRunId: null,
        cleanupStatus: null, metadata: { newOwner: true } }).where(eq(environmentLeases.id, lease.id));
      return outcome === "unconfirmed" ? undefined : { providerLeaseId: lease.providerLeaseId, state: "stopped" };
    });
    const workerManager = { isRunning: (id: string) => !(outcome === "owner_stopped" && id === seeded.pluginId), call,
      getWorker: () => ({ supportedMethods: outcome === "no_stop_capability" ? ["environmentReleaseLease", "environmentDestroyLease"] : ["environmentStopLease", "environmentDestroyLease"] }),
    } as unknown as PluginWorkerManager;
    const restarted = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
    await heartbeatService(db, { environmentRuntime: restarted }).sweepPendingCleanupLeases({ backoffMs: 0 });
    expect(call.mock.calls.some((entry) => entry[1] === "environmentDestroyLease")).toBe(false);
    const persisted = (await environmentService(db).getLeaseById(lease.id))!;
    if (["stopped", "duplicate_key", "legacy_intent"].includes(outcome)) {
      expect(call).toHaveBeenCalledWith(seeded.pluginId, "environmentStopLease", expect.objectContaining({ providerLeaseId: lease.providerLeaseId, cancelActiveWork: true }), expect.any(Number));
      expect(persisted).toMatchObject({ status: "released", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "stopped", runId: lease.heartbeatRunId } } });
      expect(persisted.metadata?.nativeWorkspaceExportResume).toBeUndefined();
    } else if (outcome === "late_receipt") {
      expect(persisted).toMatchObject({ status: "active", heartbeatRunId: null, cleanupStatus: null, metadata: { newOwner: true } });
    } else {
      expect(persisted).toMatchObject({ status: "pending_cleanup", cleanupStatus: "failed" });
      expect(persisted.metadata?.remoteExecutionTermination).toBeUndefined();
      if (["foreign_marker", "foreign_plugin_pin", "v2_missing_pin", "competing_owner", "retained_owner", "no_stop_capability", "live_request", "owner_stopped", "owner_driver_changed", "owner_missing"].includes(outcome)) expect(call).not.toHaveBeenCalled();
      if (["owner_stopped", "owner_driver_changed", "owner_missing"].includes(outcome)) {
        expect(persisted.metadata?.pendingCleanupAttemptId).toBe(requestId);
      }
    }
  });

  it.each(["stopped", "stop_failed", "unconfirmed", "restart", "committed_release", "inflight"])("preserves terminal ephemeral export work through release and restart: %s", async outcome => {
    const seeded = await seedReusablePluginSandboxLease("paperclip_runner");
    const lease = seeded.reusableLease, requestId = randomUUID();
    await db.update(heartbeatRuns).set({ status: outcome === "committed_release" ? "succeeded" : "failed" }).where(eq(heartbeatRuns.id, seeded.runId));
    await db.update(environmentLeases).set({ status: outcome === "committed_release" ? "active" : "pending_cleanup", leasePolicy: "ephemeral", cleanupStatus: "failed", metadata: {
      ...lease.metadata, reuseLease: false,
      nativeWorkspaceExportResume: { schema: "paperclip.workspace-export-resume.v2", purpose: "terminal_export", requestId,
        companyId: seeded.companyId, pluginId: seeded.pluginId, runId: seeded.runId, resultId: randomUUID(),
        leaseId: lease.id, provider: lease.provider, providerLeaseId: lease.providerLeaseId },
      pendingCleanupAttemptId: requestId, pendingCleanupInFlight: outcome === "inflight",
      pendingCleanupLeaseExpiresAtMs: outcome === "inflight" ? Date.now() + 60_000 : 0,
    } }).where(eq(environmentLeases.id, lease.id));
    let calls = 0;
    const call = vi.fn(async (_id: string, method: string, params: any) => {
      expect(method).toBe("environmentStopLease");
      expect(params).toMatchObject({ providerLeaseId: lease.providerLeaseId, resourceDisposition: "stop_and_retain", config: { reuseLease: false } });
      if (++calls === 1 && outcome === "stop_failed") throw new Error("injected stop failure");
      if (calls === 1 && outcome === "unconfirmed") return undefined;
      return { providerLeaseId: lease.providerLeaseId, state: "stopped" };
    });
    const runtime = () => environmentRuntimeService(db, { pluginWorkerManager: { isRunning: () => true, call,
      getWorker: () => ({ supportedMethods: ["environmentStopLease", "environmentDestroyLease"] }),
    } as unknown as PluginWorkerManager });
    if (outcome === "restart") await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 });
    else await runtime().releaseRunLeases(seeded.runId, "failed", undefined, "stop_and_retain");
    if (outcome === "inflight") { expect(call).not.toHaveBeenCalled(); return; }
    if (["stop_failed", "unconfirmed"].includes(outcome)) {
      expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "pending_cleanup", metadata: { pendingCleanupInFlight: false } });
      await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 });
    }
    expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "released", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "stopped" } } });
    expect(call.mock.calls.every(entry => entry[1] === "environmentStopLease")).toBe(true);
  });

  it("does not allocate a native-runner replacement without a verified backup", async () => {
    const seeded = await seedReusablePluginSandboxLease("paperclip_runner");
    const workerManager = {
      isRunning: vi.fn((id: string) => id === seeded.pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentResumeLease") {
          return { providerLeaseId: null, metadata: { expired: true } };
        }
        if (method === "environmentDestroyLease") return undefined;
        if (method === "environmentAcquireLease") {
          throw new Error("replacement must not be allocated");
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({
        supportedMethods: [
          "environmentResumeLease",
          "environmentReleaseLease",
          "environmentDestroyLease",
        ],
      })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    await expect(runtimeWithPlugin.acquireRunLease({
      companyId: seeded.companyId,
      environment: seeded.environment,
      issueId: null,
      agentId: seeded.agentId,
      adapterType: "paperclip_runner",
      heartbeatRunId: seeded.runId,
      persistedExecutionWorkspace: {
        id: seeded.executionWorkspaceId,
        mode: "shared_workspace",
      },
    })).rejects.toThrow("runner_harness_backup_unavailable");

    expect(workerManager.call).not.toHaveBeenCalledWith(
      seeded.pluginId,
      "environmentAcquireLease",
      expect.anything(),
      expect.anything(),
    );
    expect(workerManager.call).not.toHaveBeenCalledWith(
      seeded.pluginId,
      "environmentDestroyLease",
      expect.anything(),
      expect.anything(),
    );
  });

  it("permits native-runner replacement only after verifying the stamped backup", async () => {
    const seeded = await seedReusablePluginSandboxLease("paperclip_runner");
    const backupBase = await mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-replacement-"));
    const previousStateDirectory = process.env.PAPERCLIP_RUNNER_STATE_DIR;
    process.env.PAPERCLIP_RUNNER_STATE_DIR = backupBase;
    try {
      const normalizedSessionId = "native-replacement-session";
      const runnerInstanceId = "native-replacement-runner";
      const sessionScopeId = "native-replacement-session-scope-v2";
      const sessionRoot = path.join(
        backupBase,
        createHash("sha256").update(sessionScopeId).digest("hex"),
      );
      const current = path.join(sessionRoot, "failover-backups", "current");
      await mkdir(path.join(current, "runner"), { recursive: true });
      await mkdir(path.join(current, "codex-home", "sessions"), { recursive: true });
      await writeFile(path.join(current, "runner", "runner-state.json"), "runner-state");
      await writeFile(path.join(current, "codex-home", "sessions", "thread.jsonl"), "thread-state");
      const execution = {
        provider: { kind: "codex", model: null, approvalPolicy: "never" },
        binding: {
          companyId: seeded.companyId,
          runId: seeded.runId,
          issueId: "issue",
          agentId: seeded.agentId,
          executionWorkspaceId: seeded.executionWorkspaceId,
        },
        workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
        session: {
          normalizedSessionId,
          driverKind: "codex_app_server",
          protocolVersion: 1,
          lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null },
        },
      } as never;
      const manifest = buildNativeHarnessBackupManifest({
        backupRoot: current,
        execution,
        runnerInstanceId,
        providerSessionIdentity: {
          providerSessionId: "thread-1",
          providerBackendSessionId: "session-1",
          providerSessionIdentity: null,
        },
        sourceProviderLeaseId: seeded.reusableLease.providerLeaseId!,
      });
      const manifestPath = path.join(current, "manifest.json");
      await writeFile(manifestPath, JSON.stringify(manifest));
      const stamp = createNativeHarnessBackupStamp({
        manifestPath,
        sessionScopeId,
        authorizedProviderLeaseId: seeded.reusableLease.providerLeaseId!,
        normalizedSessionId,
        runnerInstanceId,
        completedAt: manifest.completedAt,
      });
      await environmentService(db).updateLeaseMetadata(seeded.reusableLease.id, {
        ...(seeded.reusableLease.metadata ?? {}),
        nativeHarnessBackup: stamp,
      });

      const workerManager = {
        isRunning: vi.fn((id: string) => id === seeded.pluginId),
        call: vi.fn(async (_pluginId: string, method: string) => {
          if (method === "environmentResumeLease") {
            return { providerLeaseId: null, metadata: { expired: true } };
          }
          if (method === "environmentDestroyLease") return undefined;
          if (method === "environmentAcquireLease") {
            return {
              providerLeaseId: "replacement-plugin-lease",
              metadata: {
                provider: "fake-plugin",
                image: "fake:test",
                timeoutMs: 1234,
                reuseLease: true,
                remoteCwd: "/workspace",
              },
            };
          }
          throw new Error(`Unexpected plugin method: ${method}`);
        }),
        getWorker: vi.fn(() => ({
          supportedMethods: [
            "environmentResumeLease",
            "environmentReleaseLease",
            "environmentDestroyLease",
          ],
        })),
      } as unknown as PluginWorkerManager;
      const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
      const acquired = await runtimeWithPlugin.acquireRunLease({
        companyId: seeded.companyId,
        environment: seeded.environment,
        issueId: null,
        agentId: seeded.agentId,
        adapterType: "paperclip_runner",
        heartbeatRunId: seeded.runId,
        persistedExecutionWorkspace: {
          id: seeded.executionWorkspaceId,
          mode: "shared_workspace",
        },
      });

      expect(acquired.lease.providerLeaseId).toBe("replacement-plugin-lease");
      expect(acquired.lease.metadata?.sandboxLeaseAcquisition).toEqual({
        outcome: "replacement",
        reason: "not_found",
      });
    } finally {
      if (previousStateDirectory === undefined) {
        delete process.env.PAPERCLIP_RUNNER_STATE_DIR;
      } else {
        process.env.PAPERCLIP_RUNNER_STATE_DIR = previousStateDirectory;
      }
      await rm(backupBase, { recursive: true, force: true });
    }
  });

  it("fails closed and does not resume when a worker restart drops the resume method after the capability snapshot", async () => {
    const pluginId = randomUUID();
    const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
    const providerConfig = {
      provider: "fake-plugin",
      image: "fake:test",
      timeoutMs: 1234,
      reuseLease: true,
    };
    const environment = {
      ...baseEnvironment,
      name: "Reusable Plugin Sandbox",
      driver: "sandbox",
      config: providerConfig,
    };
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: providerConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "acme.fake-sandbox-provider",
      packageName: "@acme/fake-sandbox-provider",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.fake-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Fake Sandbox Provider",
        description: "Test schema-driven provider",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            supportsReusableLeases: true,
            configSchema: {
              type: "object",
              properties: {
                image: { type: "string" },
                timeoutMs: { type: "number" },
                reuseLease: { type: "boolean" },
              },
            },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);
    const executionWorkspaceId = randomUUID();
    const projectId = randomUUID();
    await db.insert(projects).values({
      id: projectId,
      companyId,
      name: `Workspace ${projectId.slice(0, 8)}`,
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(executionWorkspaces).values({
      id: executionWorkspaceId,
      companyId,
      projectId,
      mode: "shared_workspace",
      strategyType: "project_primary",
      name: "Reusable workspace",
      status: "active",
      providerType: "local_fs",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    const staleLease = await environmentService(db).acquireLease({
      companyId,
      environmentId: environment.id,
      executionWorkspaceId,
      heartbeatRunId: runId,
      leasePolicy: "reuse_by_environment",
      provider: "fake-plugin",
      providerLeaseId: "stale-plugin-lease",
      metadata: {
        agentId,
        driver: "sandbox",
        pluginId,
        pluginKey: "acme.fake-sandbox-provider",
        sandboxProviderPlugin: true,
        provider: "fake-plugin",
        image: "fake:test",
        timeoutMs: 1234,
        reuseLease: true,
        reusableSandboxLease: {
          version: 1,
          companyId,
          environmentId: environment.id,
          executionWorkspaceId,
          agentId,
          adapterType: null,
          provider: "fake-plugin",
          runtimeFingerprint: reusableRuntimeFingerprint({
            provider: "fake-plugin",
            adapterType: null,
            config: providerConfig,
          }),
        },
      },
    });

    // The runtime reads the worker methods once to decide reuse, then does
    // asynchronous database work before the resume dispatch. A worker restart
    // in that window drops `environmentResumeLease`. The first read returns the
    // reuse verbs, so the runtime treats the lease as resumable. Every later
    // read returns the restarted worker's methods, which no longer include
    // `environmentResumeLease`.
    let getWorkerReads = 0;
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentResumeLease") {
          // A regression that reads the stale snapshot dispatches the resume
          // RPC and fails here. The live worker cannot serve the method.
          throw new Error("worker no longer advertises environmentResumeLease");
        }
        if (method === "environmentDestroyLease") {
          return undefined;
        }
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "fresh-plugin-lease",
            metadata: {
              provider: "fake-plugin",
              image: "fake:test",
              timeoutMs: 1234,
              reuseLease: true,
              remoteCwd: "/workspace",
            },
          };
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => {
        getWorkerReads += 1;
        return {
          supportedMethods:
            getWorkerReads === 1
              ? ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"]
              : ["environmentReleaseLease", "environmentDestroyLease"],
        };
      }),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    await expect(runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      agentId,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: {
        id: executionWorkspaceId,
        mode: "shared_workspace",
      },
    })).rejects.toThrow("the lease was preserved and no replacement was created");

    // The runtime re-checks the live worker before the resume dispatch. The
    // restarted worker no longer advertises `environmentResumeLease`, so the
    // runtime must not dispatch the resume RPC.
    expect(workerManager.call).not.toHaveBeenCalledWith(
      pluginId,
      "environmentResumeLease",
      expect.anything(),
      expect.anything(),
    );
    // It preserves the stale reusable lease and does not allocate a replacement.
    expect(workerManager.call).not.toHaveBeenCalledWith(
      pluginId,
      "environmentDestroyLease",
      expect.anything(),
      expect.anything(),
    );
    expect(workerManager.call).not.toHaveBeenCalledWith(
      pluginId,
      "environmentAcquireLease",
      expect.anything(),
      expect.anything(),
    );
    await expect(environmentService(db).getLeaseById(staleLease.id)).resolves.toMatchObject({
      status: "active",
    });
  });

  // Seed a reusable plugin sandbox lease that a worker created under an earlier
  // capability set. The worker restarts and no longer advertises the lifecycle
  // methods. The lease lifecycle paths must verify the live worker before they
  // dispatch a lifecycle RPC, so the runtime fails closed instead of a doomed
  // dispatch.
  async function seedStaleLifecycleReusableLease(
    leasePolicy: "reuse_by_environment" | "retain_on_failure" = "reuse_by_environment",
  ) {
    const pluginId = randomUUID();
    const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
    const providerConfig = {
      provider: "fake-plugin",
      image: "fake:test",
      timeoutMs: 1234,
      reuseLease: true,
    };
    const environment = {
      ...baseEnvironment,
      name: "Reusable Plugin Sandbox",
      driver: "sandbox",
      config: providerConfig,
    };
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: providerConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "acme.fake-sandbox-provider",
      packageName: "@acme/fake-sandbox-provider",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.fake-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Fake Sandbox Provider",
        description: "Test schema-driven provider",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            supportsReusableLeases: true,
            configSchema: { type: "object" },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);
    const executionWorkspaceId = randomUUID();
    const projectId = randomUUID();
    await db.insert(projects).values({
      id: projectId,
      companyId,
      name: `Workspace ${projectId.slice(0, 8)}`,
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(executionWorkspaces).values({
      id: executionWorkspaceId,
      companyId,
      projectId,
      mode: "shared_workspace",
      strategyType: "project_primary",
      name: "Reusable workspace",
      status: "active",
      providerType: "local_fs",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    const lease = await environmentService(db).acquireLease({
      companyId,
      environmentId: environment.id,
      executionWorkspaceId,
      heartbeatRunId: runId,
      leasePolicy,
      provider: "fake-plugin",
      providerLeaseId: "stale-lifecycle-lease",
      metadata: {
        agentId,
        driver: "sandbox",
        pluginId,
        pluginKey: "acme.fake-sandbox-provider",
        sandboxProviderPlugin: true,
        provider: "fake-plugin",
        image: "fake:test",
        timeoutMs: 1234,
        reuseLease: true,
      },
    });

    // The worker is running, but its discovery list dropped the reusable-lease
    // lifecycle methods. `call` throws on any lifecycle RPC so a regression that
    // dispatches one fails the test through the `not.toHaveBeenCalledWith` check.
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentAcquireLease", "environmentExecute"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    return { pluginId, environment, runId, lease, workerManager, runtimeWithPlugin };
  }

  it("routes release to pending_cleanup when the worker no longer advertises the release lifecycle method", async () => {
    const { pluginId, lease, workerManager, runtimeWithPlugin } = await seedStaleLifecycleReusableLease();

    const released = await runtimeWithPlugin.releaseRunLeases(lease.heartbeatRunId!);

    expect(released).toHaveLength(1);
    expect(workerManager.call).not.toHaveBeenCalledWith(
      pluginId,
      "environmentReleaseLease",
      expect.anything(),
      expect.anything(),
    );
    // The failed release verification must enter the pending-cleanup retry flow.
    // The reaper sweeps only `pending_cleanup` leases, so a `released` status
    // here would strand the still-active provider resource.
    await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({
      status: "pending_cleanup",
      cleanupStatus: "failed",
      failureReason: "release_cleanup_failed",
    });
  });

  it("retains a retain_on_failure lease on failed release instead of routing to pending_cleanup", async () => {
    const { lease, runtimeWithPlugin } = await seedStaleLifecycleReusableLease("retain_on_failure");

    const released = await runtimeWithPlugin.releaseRunLeases(lease.heartbeatRunId!, "failed");

    expect(released).toHaveLength(1);
    // A retain_on_failure lease keeps the provider resource for reuse. The
    // reaper destroys `pending_cleanup` leases, so the retained lease must not
    // enter that flow even when the release verification fails.
    await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({
      status: "retained",
      cleanupStatus: "failed",
    });
  });

  it("destroys a failed retained lease when the agent stops", async () => {
    const { lease, workerManager, runtimeWithPlugin, runId } = await seedStaleLifecycleReusableLease("retain_on_failure");
    await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId));
    await runtimeWithPlugin.releaseRunLeases(runId, "failed");
    await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({ status: "retained", cleanupStatus: "failed" });
    vi.mocked(workerManager.getWorker).mockReturnValue({ supportedMethods: ["environmentDestroyLease"] } as never);
    vi.mocked(workerManager.call).mockResolvedValue({ providerLeaseId: lease.providerLeaseId, state: "destroyed" });
    const { heartbeatService } = await import("../services/heartbeat.js");
    const stopped = await heartbeatService(db, { pluginWorkerManager: workerManager })
      .stopInvocationsForAgents([lease.metadata!.agentId as string], "Agent lifecycle stop requested");
    expect(stopped).toBe(true);
    expect(workerManager.call).toHaveBeenCalledWith(expect.any(String), "environmentDestroyLease", expect.objectContaining({ providerLeaseId: lease.providerLeaseId }), expect.any(Number));
    await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({ status: "expired", cleanupStatus: "success" });
  });

  it("fails closed on expiry destruction when the worker no longer advertises the destroy lifecycle method", async () => {
    const { pluginId, lease, workerManager, runtimeWithPlugin } = await seedStaleLifecycleReusableLease();

    await runtimeWithPlugin.releaseRunLeases(lease.heartbeatRunId!, "expired");

    expect(workerManager.call).not.toHaveBeenCalledWith(
      pluginId,
      "environmentDestroyLease",
      expect.anything(),
      expect.anything(),
    );
    await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({
      status: "pending_cleanup",
      cleanupStatus: "failed",
    });
  });

  it("does not resume released reusable plugin sandbox leases after provider config drift", async () => {
    const pluginId = randomUUID();
    const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
    const providerConfig = {
      provider: "fake-plugin",
      image: "template-a",
      timeoutMs: 1234,
      reuseLease: true,
    };
    const environment = {
      ...baseEnvironment,
      name: "Reusable Plugin Sandbox",
      driver: "sandbox",
      config: providerConfig,
    };
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: providerConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "acme.fake-sandbox-provider",
      packageName: "@acme/fake-sandbox-provider",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.fake-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Fake Sandbox Provider",
        description: "Test schema-driven provider",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            supportsReusableLeases: true,
            configSchema: {
              type: "object",
              properties: {
                image: { type: "string" },
                timeoutMs: { type: "number" },
                reuseLease: { type: "boolean" },
              },
            },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);
    const executionWorkspaceId = randomUUID();
    const projectId = randomUUID();
    await db.insert(projects).values({
      id: projectId,
      companyId,
      name: `Workspace ${projectId.slice(0, 8)}`,
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(executionWorkspaces).values({
      id: executionWorkspaceId,
      companyId,
      projectId,
      mode: "shared_workspace",
      strategyType: "project_primary",
      name: "Reusable workspace",
      status: "active",
      providerType: "local_fs",
      createdAt: new Date(),
      updatedAt: new Date(),
    });

    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string, params: any) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: `lease-${params.config.image}`,
            metadata: {
              provider: "fake-plugin",
              image: params.config.image,
              timeoutMs: params.config.timeoutMs,
              reuseLease: true,
              remoteCwd: "/workspace",
            },
          };
        }
        if (method === "environmentReleaseLease" || method === "environmentDestroyLease") {
          return undefined;
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const first = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      agentId,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: {
        id: executionWorkspaceId,
        mode: "shared_workspace",
      },
    });
    expect(first.lease.metadata?.reusableSandboxLease).toMatchObject({
      provider: "fake-plugin",
      leaseFingerprint: expect.objectContaining({
        category: "lease",
        fingerprint: expect.stringMatching(/^v1:sha256:[a-f0-9]{64}$/),
      }),
    });
    await runtimeWithPlugin.releaseRunLeases(runId);

    const nextRunId = randomUUID();
    await db.insert(heartbeatRuns).values({
      id: nextRunId,
      companyId,
      agentId,
      invocationSource: "manual",
      status: "running",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    const updatedEnvironment = {
      ...environment,
      config: {
        ...providerConfig,
        image: "template-b",
      },
    };
    await environmentService(db).update(environment.id, {
      config: updatedEnvironment.config,
    });

    const second = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment: updatedEnvironment,
      issueId: null,
      agentId,
      heartbeatRunId: nextRunId,
      persistedExecutionWorkspace: {
        id: executionWorkspaceId,
        mode: "shared_workspace",
      },
    });

    expect(second.lease.providerLeaseId).toBe("lease-template-b");
    expect(workerManager.call).not.toHaveBeenCalledWith(
      pluginId,
      "environmentResumeLease",
      expect.anything(),
      expect.anything(),
    );
    expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease", expect.objectContaining({
      providerLeaseId: "lease-template-a",
    }), 31234);
    await expect(environmentService(db).getLeaseById(first.lease.id)).resolves.toMatchObject({
      status: "expired",
      cleanupStatus: "success",
      failureReason: "lease_fingerprint_mismatch",
    });
  });

  it("does not resume released reusable plugin sandbox leases after secret version drift", async () => {
    const pluginId = randomUUID();
    const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
    const apiSecret = await secretService(db).create(companyId, {
      name: `secure-plugin-api-key-${randomUUID()}`,
      provider: "local_encrypted",
      value: "resolved-provider-key",
    });
    const providerConfig = {
      provider: "secure-plugin",
      template: "base",
      apiKey: apiSecret.id,
      timeoutMs: 1234,
      reuseLease: true,
    };
    const environment = {
      ...baseEnvironment,
      name: "Secure Plugin Sandbox",
      driver: "sandbox",
      config: providerConfig,
    };
    await secretService(db).createBinding({
      companyId,
      secretId: apiSecret.id,
      targetType: "environment",
      targetId: environment.id,
      configPath: "apiKey",
    });
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: providerConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "acme.secure-sandbox-provider",
      packageName: "@acme/secure-sandbox-provider",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.secure-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Secure Sandbox Provider",
        description: "Test schema-driven provider",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "secure-plugin",
            kind: "sandbox_provider",
            displayName: "Secure Sandbox",
            supportsReusableLeases: true,
            configSchema: {
              type: "object",
              properties: {
                template: { type: "string" },
                apiKey: { type: "string", format: "secret-ref" },
                timeoutMs: { type: "number" },
                reuseLease: { type: "boolean" },
              },
            },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);
    const executionWorkspaceId = randomUUID();
    const projectId = randomUUID();
    await db.insert(projects).values({
      id: projectId,
      companyId,
      name: `Workspace ${projectId.slice(0, 8)}`,
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(executionWorkspaces).values({
      id: executionWorkspaceId,
      companyId,
      projectId,
      mode: "shared_workspace",
      strategyType: "project_primary",
      name: "Reusable workspace",
      status: "active",
      providerType: "local_fs",
      createdAt: new Date(),
      updatedAt: new Date(),
    });

    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string, params: any) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: `lease-${params.config.apiKey}`,
            metadata: {
              provider: "secure-plugin",
              template: params.config.template,
              apiKey: params.config.apiKey,
              timeoutMs: params.config.timeoutMs,
              reuseLease: true,
              remoteCwd: "/workspace",
            },
          };
        }
        if (method === "environmentReleaseLease" || method === "environmentDestroyLease") {
          return undefined;
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const first = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      agentId,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: {
        id: executionWorkspaceId,
        mode: "shared_workspace",
      },
    });
    await runtimeWithPlugin.releaseRunLeases(runId);
    await secretService(db).rotate(apiSecret.id, { value: "rotated-provider-key" });

    const nextRunId = randomUUID();
    await db.insert(heartbeatRuns).values({
      id: nextRunId,
      companyId,
      agentId,
      invocationSource: "manual",
      status: "running",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    const second = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      agentId,
      heartbeatRunId: nextRunId,
      persistedExecutionWorkspace: {
        id: executionWorkspaceId,
        mode: "shared_workspace",
      },
    });

    expect(second.lease.providerLeaseId).toBe("lease-rotated-provider-key");
    expect(workerManager.call).not.toHaveBeenCalledWith(
      pluginId,
      "environmentResumeLease",
      expect.anything(),
      expect.anything(),
    );
    expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease", expect.objectContaining({
      providerLeaseId: "lease-resolved-provider-key",
    }), 31234);
    await expect(environmentService(db).getLeaseById(first.lease.id)).resolves.toMatchObject({
      status: "expired",
      cleanupStatus: "success",
      failureReason: "lease_fingerprint_mismatch",
    });
    const firstMetadata = JSON.stringify(first.lease.metadata);
    expect(firstMetadata).not.toContain("resolved-provider-key");
    expect(firstMetadata).not.toContain("rotated-provider-key");
  });

  it("preserves active reusable sandbox leases held by another running run", async () => {
    const { pluginId, companyId, agentId, environment, executionWorkspaceId, reusableLease } =
      await seedReusablePluginSandboxLease();
    const nextRunId = randomUUID();
    await db.insert(heartbeatRuns).values({
      id: nextRunId,
      companyId,
      agentId,
      invocationSource: "manual",
      status: "running",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "fresh-plugin-lease",
            metadata: {
              provider: "fake-plugin",
              image: "fake:test",
              timeoutMs: 1234,
              reuseLease: true,
              remoteCwd: "/workspace",
            },
          };
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const acquired = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      agentId,
      heartbeatRunId: nextRunId,
      persistedExecutionWorkspace: {
        id: executionWorkspaceId,
        mode: "shared_workspace",
      },
    });

    expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease");
    expect(workerManager.call).toHaveBeenCalledOnce();
    expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.objectContaining({
      agentId,
      executionWorkspaceId,
      runId: nextRunId,
    }), 31234);
    await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
      status: "active",
      cleanupStatus: null,
    });
  });

  it("does not retain or resume plugin-backed sandbox leases unless the provider opts in", async () => {
    const pluginId = randomUUID();
    const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
    const providerConfig = {
      provider: "fake-plugin",
      image: "fake:test",
      timeoutMs: 1234,
      reuseLease: true,
    };
    const environment = {
      ...baseEnvironment,
      name: "Non-reusable Plugin Sandbox",
      driver: "sandbox",
      config: providerConfig,
    };
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: providerConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "acme.non-reusable-sandbox-provider",
      packageName: "@acme/non-reusable-sandbox-provider",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.non-reusable-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Non-reusable Sandbox Provider",
        description: "Test provider without reusable lease support",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            configSchema: { type: "object" },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);
    const executionWorkspaceId = randomUUID();
    const projectId = randomUUID();
    await db.insert(projects).values({
      id: projectId,
      companyId,
      name: `Workspace ${projectId.slice(0, 8)}`,
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(executionWorkspaces).values({
      id: executionWorkspaceId,
      companyId,
      projectId,
      mode: "shared_workspace",
      strategyType: "project_primary",
      name: "Non-reusable workspace",
      status: "active",
      providerType: "local_fs",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await environmentService(db).acquireLease({
      companyId,
      environmentId: environment.id,
      executionWorkspaceId,
      heartbeatRunId: runId,
      leasePolicy: "reuse_by_environment",
      provider: "fake-plugin",
      providerLeaseId: "old-plugin-lease",
      metadata: {
        agentId,
        provider: "fake-plugin",
        image: "fake:test",
        timeoutMs: 1234,
        reuseLease: true,
        reusableSandboxLease: {
          version: 1,
          companyId,
          environmentId: environment.id,
          executionWorkspaceId,
          agentId,
          adapterType: null,
          provider: "fake-plugin",
          runtimeFingerprint: reusableRuntimeFingerprint({
            provider: "fake-plugin",
            adapterType: null,
            config: providerConfig,
          }),
        },
      },
    });

    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "fresh-plugin-lease",
            metadata: {
              provider: "fake-plugin",
              image: "fake:test",
              timeoutMs: 1234,
              reuseLease: true,
              remoteCwd: "/workspace",
            },
          };
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const acquired = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      agentId,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: {
        id: executionWorkspaceId,
        mode: "shared_workspace",
      },
    });

    expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease");
    expect(acquired.lease.leasePolicy).toBe("ephemeral");
    expect(workerManager.call).toHaveBeenCalledTimes(1);
    expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234);
  });

  it("does not resume a lease when the nested capability disables reusable leases", async () => {
    // The provider declares the legacy `supportsReusableLeases: true` flag but
    // the nested `sandboxCapabilities.reusableLeases: false`. The nested value
    // wins through the capability contract, so acquisition must acquire a fresh
    // lease and must never resume the existing reusable lease.
    const pluginId = randomUUID();
    const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
    const providerConfig = {
      provider: "fake-plugin",
      image: "fake:test",
      timeoutMs: 1234,
      reuseLease: true,
    };
    const environment = {
      ...baseEnvironment,
      name: "Nested-disabled Plugin Sandbox",
      driver: "sandbox",
      config: providerConfig,
    };
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: providerConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "acme.nested-disabled-sandbox-provider",
      packageName: "@acme/nested-disabled-sandbox-provider",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.nested-disabled-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Nested-disabled Sandbox Provider",
        description: "Test provider with a legacy flag and a disabled nested capability",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            supportsReusableLeases: true,
            sandboxCapabilities: { reusableLeases: false },
            configSchema: {
              type: "object",
              properties: {
                image: { type: "string" },
                timeoutMs: { type: "number" },
                reuseLease: { type: "boolean" },
              },
            },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);
    const executionWorkspaceId = randomUUID();
    const projectId = randomUUID();
    await db.insert(projects).values({
      id: projectId,
      companyId,
      name: `Workspace ${projectId.slice(0, 8)}`,
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(executionWorkspaces).values({
      id: executionWorkspaceId,
      companyId,
      projectId,
      mode: "shared_workspace",
      strategyType: "project_primary",
      name: "Nested-disabled workspace",
      status: "active",
      providerType: "local_fs",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await environmentService(db).acquireLease({
      companyId,
      environmentId: environment.id,
      executionWorkspaceId,
      heartbeatRunId: runId,
      leasePolicy: "reuse_by_environment",
      provider: "fake-plugin",
      providerLeaseId: "old-plugin-lease",
      metadata: {
        agentId,
        provider: "fake-plugin",
        image: "fake:test",
        timeoutMs: 1234,
        reuseLease: true,
        reusableSandboxLease: {
          version: 1,
          companyId,
          environmentId: environment.id,
          executionWorkspaceId,
          agentId,
          adapterType: null,
          provider: "fake-plugin",
          runtimeFingerprint: reusableRuntimeFingerprint({
            provider: "fake-plugin",
            adapterType: null,
            config: providerConfig,
          }),
        },
      },
    });

    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "fresh-plugin-lease",
            metadata: {
              provider: "fake-plugin",
              image: "fake:test",
              timeoutMs: 1234,
              reuseLease: true,
              remoteCwd: "/workspace",
            },
          };
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const acquired = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      agentId,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: {
        id: executionWorkspaceId,
        mode: "shared_workspace",
      },
    });

    expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease");
    expect(acquired.lease.leasePolicy).toBe("ephemeral");
    expect(workerManager.call).toHaveBeenCalledTimes(1);
    expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234);
    expect(workerManager.call).not.toHaveBeenCalledWith(
      pluginId,
      "environmentResumeLease",
      expect.anything(),
      expect.anything(),
    );
  });

  it("fails closed and does not resume when the worker does not verify the reuse methods", async () => {
    // The provider declares `reusableLeases: true`, but its worker advertises
    // neither `environmentResumeLease` nor `environmentReleaseLease`. The runtime
    // must not resume or reuse the lease, because it cannot dispatch a resume or
    // a release the worker does not serve. It acquires a fresh ephemeral lease
    // and leaves the old reusable lease untouched.
    const pluginId = randomUUID();
    const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment();
    const providerConfig = {
      provider: "fake-plugin",
      image: "fake:test",
      timeoutMs: 1234,
      reuseLease: true,
    };
    const environment = {
      ...baseEnvironment,
      name: "Unverified-worker Plugin Sandbox",
      driver: "sandbox",
      config: providerConfig,
    };
    await environmentService(db).update(environment.id, {
      driver: "sandbox",
      name: environment.name,
      config: providerConfig,
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "acme.unverified-worker-sandbox-provider",
      packageName: "@acme/unverified-worker-sandbox-provider",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.unverified-worker-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Unverified-worker Sandbox Provider",
        description: "Test provider that declares reusable leases but whose worker lacks the reuse methods",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            sandboxCapabilities: { reusableLeases: true },
            configSchema: {
              type: "object",
              properties: {
                image: { type: "string" },
                timeoutMs: { type: "number" },
                reuseLease: { type: "boolean" },
              },
            },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);
    const executionWorkspaceId = randomUUID();
    const projectId = randomUUID();
    await db.insert(projects).values({
      id: projectId,
      companyId,
      name: `Workspace ${projectId.slice(0, 8)}`,
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(executionWorkspaces).values({
      id: executionWorkspaceId,
      companyId,
      projectId,
      mode: "shared_workspace",
      strategyType: "project_primary",
      name: "Unverified-worker workspace",
      status: "active",
      providerType: "local_fs",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    const existingLease = await environmentService(db).acquireLease({
      companyId,
      environmentId: environment.id,
      executionWorkspaceId,
      heartbeatRunId: runId,
      leasePolicy: "reuse_by_environment",
      provider: "fake-plugin",
      providerLeaseId: "old-plugin-lease",
      metadata: {
        agentId,
        provider: "fake-plugin",
        image: "fake:test",
        timeoutMs: 1234,
        reuseLease: true,
        reusableSandboxLease: {
          version: 1,
          companyId,
          environmentId: environment.id,
          executionWorkspaceId,
          agentId,
          adapterType: null,
          provider: "fake-plugin",
          runtimeFingerprint: reusableRuntimeFingerprint({
            provider: "fake-plugin",
            adapterType: null,
            config: providerConfig,
          }),
        },
      },
    });

    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "fresh-plugin-lease",
            metadata: {
              provider: "fake-plugin",
              image: "fake:test",
              timeoutMs: 1234,
              reuseLease: true,
              remoteCwd: "/workspace",
            },
          };
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      // The worker verifies only `environmentExecute`. It advertises neither
      // reuse verb, so the reusable-lease capability fails closed.
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentExecute"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const acquired = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      agentId,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: {
        id: executionWorkspaceId,
        mode: "shared_workspace",
      },
    });

    expect(acquired.lease.providerLeaseId).toBe("fresh-plugin-lease");
    expect(acquired.lease.leasePolicy).toBe("ephemeral");
    expect(workerManager.call).toHaveBeenCalledTimes(1);
    expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything(), 31234);
    expect(workerManager.call).not.toHaveBeenCalledWith(
      pluginId,
      "environmentResumeLease",
      expect.anything(),
      expect.anything(),
    );
    // The runtime leaves the old reusable lease untouched: it neither resumes
    // nor destroys a lease it cannot serve.
    await expect(environmentService(db).getLeaseById(existingLease.id)).resolves.toMatchObject({
      status: "active",
      leasePolicy: "reuse_by_environment",
    });
  });

  it.each([
    { status: "active", scope: "workspace" },
    { status: "failed", scope: "workspace" },
    { status: "failed", scope: "issue" },
  ] as const)("destroys $status reusable plugin-backed sandbox leases scoped to an $scope", async ({ status, scope }) => {
    const { pluginId, companyId, runId, executionWorkspaceId, reusableLease } =
      await seedReusablePluginSandboxLease();
    await db
      .update(heartbeatRuns)
      .set({ status: status === "failed" ? "failed" : "succeeded" })
      .where(eq(heartbeatRuns.id, runId));
    if (status === "failed") await environmentService(db).releaseLease(reusableLease.id, "failed");
    const issueId = randomUUID();
    if (scope === "issue") {
      await db.insert(issues).values({ id: issueId, companyId, title: "Failed reusable lease cleanup", status: "done", priority: "medium" });
      await db.update(environmentLeases).set({ issueId }).where(eq(environmentLeases.id, reusableLease.id));
    }
    const selection = scope === "issue" ? { issueId } : { executionWorkspaceId };

    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentDestroyLease") {
          return undefined;
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    // A known issue/workspace id does not authorize another company's teardown.
    expect(await runtimeWithPlugin.destroyReusableSandboxLeases({ companyId: randomUUID(), ...selection })).toEqual([]);
    expect(workerManager.call).not.toHaveBeenCalled();
    const destroyed = await runtimeWithPlugin.destroyReusableSandboxLeases({
      companyId,
      ...selection,
      failureReason: "execution_workspace_closed",
    });

    expect(destroyed).toHaveLength(1);
    expect(destroyed[0]?.lease.id).toBe(reusableLease.id);
    expect(destroyed[0]?.lease.status).toBe("expired");
    expect(workerManager.call).toHaveBeenCalledWith(
      pluginId,
      "environmentDestroyLease",
      expect.objectContaining({
        driverKey: "fake-plugin",
        providerLeaseId: "reusable-plugin-lease",
      }),
      31234,
    );
    await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
      status: "expired",
      failureReason: "execution_workspace_closed",
      cleanupStatus: "success",
    });
  });

  it.each(["active", "failed"] as const)("does not destroy a %s scoped reusable lease while its run is active", async (status) => {
    const { pluginId, companyId, executionWorkspaceId, reusableLease } =
      await seedReusablePluginSandboxLease();
    if (status === "failed") await environmentService(db).releaseLease(reusableLease.id, "failed");
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async () => undefined),
      getWorker: vi.fn(() => ({
        supportedMethods: [
          "environmentResumeLease",
          "environmentReleaseLease",
          "environmentDestroyLease",
        ],
      })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, {
      pluginWorkerManager: workerManager,
    });

    const destroyed = await runtimeWithPlugin.destroyReusableSandboxLeases({
      companyId,
      executionWorkspaceId,
      failureReason: "issue_terminal_done",
    });

    expect(destroyed).toEqual([]);
    expect(workerManager.call).not.toHaveBeenCalled();
    await expect(
      environmentService(db).getLeaseById(reusableLease.id),
    ).resolves.toMatchObject({ status });
  });

  it.each(["issue", "workspace", "environment"] as const)(
    "durably claims failed reusable cleanup before %s teardown and recovers a provider failure",
    async (scope) => {
      const seeded = await seedReusablePluginSandboxLease();
      await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId));
      await environmentService(db).releaseLease(seeded.reusableLease.id, "failed");
      const issueId = randomUUID();
      if (scope === "issue") {
        await db.insert(issues).values({ id: issueId, companyId: seeded.companyId, title: "Cleanup crash window", status: "done" });
        await db.update(environmentLeases).set({ issueId }).where(eq(environmentLeases.id, seeded.reusableLease.id));
      }
      let enterProvider!: () => void;
      let finishProvider!: () => void;
      const entered = new Promise<void>((resolve) => { enterProvider = resolve; });
      const finish = new Promise<void>((resolve) => { finishProvider = resolve; });
      let unavailable = true;
      const call = vi.fn(async (_id: string, method: string) => {
        if (method !== "environmentDestroyLease") throw new Error(`Unexpected method ${method}`);
        if (unavailable) {
          enterProvider();
          await finish;
          throw new Error("Provider response lost");
        }
        return { providerLeaseId: seeded.reusableLease.providerLeaseId, state: "destroyed" };
      });
      const makeRuntime = () => environmentRuntimeService(db, { pluginWorkerManager: {
        isRunning: () => true, call,
        getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }),
      } as unknown as PluginWorkerManager });
      const runtime = makeRuntime();
      const close = (service: ReturnType<typeof makeRuntime>) => scope === "environment"
        ? service.destroyReusableSandboxLeasesForEnvironment({ environmentId: seeded.environment.id })
        : service.destroyReusableSandboxLeases({ companyId: seeded.companyId,
          ...(scope === "issue" ? { issueId } : { executionWorkspaceId: seeded.executionWorkspaceId }) });
      const closing = close(runtime);
      try {
        await entered;
        // This independently persisted row is what survives controller loss
        // during the provider call. No provider receipt has arrived yet.
        expect(await environmentService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({
          status: "pending_cleanup", cleanupStatus: "failed", companyId: seeded.companyId,
          providerLeaseId: seeded.reusableLease.providerLeaseId,
          metadata: { pendingCleanupAttemptId: expect.any(String), pendingCleanupInFlight: true,
            pendingCleanupLeaseExpiresAtMs: expect.any(Number) },
        });
        await close(makeRuntime());
        expect(await heartbeatService(db, { environmentRuntime: makeRuntime() }).sweepPendingCleanupLeases({ backoffMs: 0 }))
          .toEqual({ swept: 0, destroyed: 0, capped: 0 });
        expect(call).toHaveBeenCalledOnce();
      } finally {
        finishProvider();
        await closing;
      }
      expect(await environmentService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({
        status: "pending_cleanup", metadata: { pendingCleanupInFlight: false },
      });
      unavailable = false;
      expect(await heartbeatService(db, { environmentRuntime: makeRuntime() }).sweepPendingCleanupLeases({ backoffMs: 0 }))
        .toEqual({ swept: 1, destroyed: 1, capped: 0 });
      expect(call).toHaveBeenCalledTimes(2);
      expect(await environmentService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({
        status: "expired", cleanupStatus: "success", metadata: { remoteExecutionTermination: { state: "destroyed" } },
      });
    },
  );

  it.each(["queued", "scheduled_retry", "running"] as const)(
    "fences scoped cleanup when its holding run becomes %s after selection", async (status) => {
      const seeded = await seedReusablePluginSandboxLease();
      await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId));
      await environmentService(db).releaseLease(seeded.reusableLease.id, "failed");
      const originalService = environmentsModule.environmentService;
      const serviceSpy = vi.spyOn(environmentsModule, "environmentService").mockImplementation((client) => {
        const service = originalService(client);
        return { ...service, getById: async (id: string) => {
          const environment = await service.getById(id);
          if (id === seeded.environment.id) {
            await db.update(heartbeatRuns).set({ status }).where(eq(heartbeatRuns.id, seeded.runId));
          }
          return environment;
        } };
      });
      const call = vi.fn(async () => undefined);
      const runtime = environmentRuntimeService(db, { pluginWorkerManager: {
        isRunning: () => true, call, getWorker: () => ({ supportedMethods: ["environmentDestroyLease"] }),
      } as unknown as PluginWorkerManager });
      try {
        expect(await runtime.destroyReusableSandboxLeases({ companyId: seeded.companyId,
          executionWorkspaceId: seeded.executionWorkspaceId })).toEqual([]);
        expect(call).not.toHaveBeenCalled();
        expect(await originalService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({ status: "failed" });
      } finally {
        serviceSpy.mockRestore();
      }
    },
  );

  it("destroys reusable plugin-backed sandbox leases scoped to an environment", async () => {
    const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease();
    // The holding run is finished, so the reservation is stale and destroyable.
    await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, runId));

    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentDestroyLease") {
          return { providerLeaseId: reusableLease.providerLeaseId, state: "destroyed" };
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const result = await runtimeWithPlugin.destroyReusableSandboxLeasesForEnvironment({
      environmentId: reusableLease.environmentId!,
      failureReason: "environment_deleted",
    });

    expect(result).toEqual({ destroyed: 1, failed: 0, skippedLiveRun: 0 });
    expect(workerManager.call).toHaveBeenCalledWith(
      pluginId,
      "environmentDestroyLease",
      expect.objectContaining({
        driverKey: "fake-plugin",
        providerLeaseId: "reusable-plugin-lease",
      }),
      31234,
    );
    await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
      status: "expired",
      failureReason: "environment_deleted",
      cleanupStatus: "success",
      metadata: { remoteExecutionTermination: { schema: "paperclip.remote-termination.v1",
        leaseId: reusableLease.id, runId, providerLeaseId: reusableLease.providerLeaseId, state: "destroyed" } },
    });
  });

  it("destroys a failed reusable sandbox after successful release before deleting its environment", async () => {
    const { pluginId, companyId, runId, environment, reusableLease } = await seedReusablePluginSandboxLease();
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentReleaseLease") {
          return { providerLeaseId: reusableLease.providerLeaseId, state: "stopped" };
        }
        if (method === "environmentDestroyLease") {
          // The provider call must already have a durable cleanup reference.
          expect(await environmentService(db).getLeaseById(reusableLease.id)).toMatchObject({
            status: "pending_cleanup", environmentId: environment.id,
          });
          return { providerLeaseId: reusableLease.providerLeaseId, state: "destroyed" };
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
    await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, runId));

    const released = await runtimeWithPlugin.releaseRunLeases(runId, "failed", undefined, undefined, true);
    expect(released[0]?.lease).toMatchObject({
      id: reusableLease.id, status: "failed", cleanupStatus: "success",
      metadata: { remoteExecutionTermination: { state: "stopped" } },
    });
    const result = await runtimeWithPlugin.destroyReusableSandboxLeasesForEnvironment({
      environmentId: environment.id, failureReason: "environment_deleted",
    });
    expect(result).toEqual({ destroyed: 1, failed: 0, skippedLiveRun: 0 });
    expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease",
      expect.objectContaining({ companyId, environmentId: environment.id, providerLeaseId: reusableLease.providerLeaseId }),
      31234);
    await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
      status: "expired", cleanupStatus: "success",
      metadata: { remoteExecutionTermination: { state: "destroyed" } },
    });
    expect((await environmentService(db).removeIfDeletable(environment.id))?.id).toBe(environment.id);
  });

  it.each(["active", "failed"] as const)("keeps a %s reusable lease held by an in-flight run out of the environment-scoped destroy", async (status) => {
    const { pluginId, reusableLease } = await seedReusablePluginSandboxLease();
    // seedEnvironment leaves the holding run in `running` status.
    if (status === "failed") await environmentService(db).releaseLease(reusableLease.id, "failed");

    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async () => undefined),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const result = await runtimeWithPlugin.destroyReusableSandboxLeasesForEnvironment({
      environmentId: reusableLease.environmentId!,
      failureReason: "environment_deleted",
    });

    expect(result).toEqual({ destroyed: 0, failed: 0, skippedLiveRun: 1 });
    expect(workerManager.call).not.toHaveBeenCalled();
    // The lease keeps its reusable status, so the delete guard still blocks.
    await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
      status,
      leasePolicy: "reuse_by_environment",
    });
  });

  it("sweeps reusable cleanup with the configuration recorded on the lease", async () => {
    const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease();
    const environmentsSvc = environmentService(db);
    await environmentsSvc.releaseLease(reusableLease.id, "pending_cleanup", {
      failureReason: "release_cleanup_failed",
      cleanupStatus: "failed",
    });

    // Re-point the environment after the reusable sandbox was provisioned.
    // The pending-cleanup sweep still enters destroyRunLease because the
    // environment exists, but destroy must target the provider and config
    // captured on the lease rather than this current configuration.
    await environmentsSvc.update(environment.id, {
      config: {
        provider: "replacement-provider",
        image: "replacement:test",
        timeoutMs: 9999,
        reuseLease: true,
      },
    });

    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentDestroyLease") return undefined;
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({
        supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"],
      })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
    const heartbeat = heartbeatService(db, { environmentRuntime: runtimeWithPlugin });

    const result = await heartbeat.sweepPendingCleanupLeases({ backoffMs: 0 });

    expect(result).toEqual({ swept: 1, destroyed: 1, capped: 0 });
    expect(workerManager.call).toHaveBeenCalledWith(
      pluginId,
      "environmentDestroyLease",
      expect.objectContaining({
        driverKey: "fake-plugin",
        environmentId: environment.id,
        providerLeaseId: "reusable-plugin-lease",
        config: expect.objectContaining({
          image: "fake:test",
          timeoutMs: 1234,
          reuseLease: true,
        }),
      }),
      31234,
    );
    const destroyInput = vi.mocked(workerManager.call).mock.calls[0]?.[2] as {
      config?: Record<string, unknown>;
    };
    expect(destroyInput.config).not.toMatchObject({
      image: "replacement:test",
      timeoutMs: 9999,
    });
    await expect(environmentsSvc.getLeaseById(reusableLease.id)).resolves.toMatchObject({
      status: "expired",
      cleanupStatus: "success",
    });
  });

  it("retries reusable plugin-backed sandbox destroy when the worker is unavailable", async () => {
    const { pluginId, companyId, executionWorkspaceId, runId, reusableLease } =
      await seedReusablePluginSandboxLease();
    await db
      .update(heartbeatRuns)
      .set({ status: "succeeded", finishedAt: new Date() })
      .where(eq(heartbeatRuns.id, runId));

    const offlineWorkerManager = {
      isRunning: vi.fn(() => false),
      call: vi.fn(),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithOfflinePlugin = environmentRuntimeService(db, {
      pluginWorkerManager: offlineWorkerManager,
    });

    const pending = await runtimeWithOfflinePlugin.destroyReusableSandboxLeases({
      companyId,
      executionWorkspaceId,
      failureReason: "execution_workspace_closed",
    });

    expect(pending).toHaveLength(1);
    expect(pending[0]?.lease.id).toBe(reusableLease.id);
    expect(pending[0]?.lease.status).toBe("pending_cleanup");
    expect(offlineWorkerManager.call).not.toHaveBeenCalled();
    await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
      status: "pending_cleanup",
      failureReason: "execution_workspace_closed",
      cleanupStatus: "failed",
    });

    const recoveredWorkerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentDestroyLease") {
          return undefined;
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithRecoveredPlugin = environmentRuntimeService(db, {
      pluginWorkerManager: recoveredWorkerManager,
    });

    const retried = await runtimeWithRecoveredPlugin.destroyReusableSandboxLeases({
      companyId,
      executionWorkspaceId,
      failureReason: "cleanup_retry",
    });

    expect(retried).toHaveLength(1);
    expect(retried[0]?.lease.id).toBe(reusableLease.id);
    expect(retried[0]?.lease.status).toBe("expired");
    expect(recoveredWorkerManager.call).toHaveBeenCalledWith(
      pluginId,
      "environmentDestroyLease",
      expect.objectContaining({
        driverKey: "fake-plugin",
        providerLeaseId: "reusable-plugin-lease",
      }),
      31234,
    );
    await expect(environmentService(db).getLeaseById(reusableLease.id)).resolves.toMatchObject({
      status: "expired",
      failureReason: "cleanup_retry",
      cleanupStatus: "success",
    });
  });

  it("resolves effective capabilities from the lease's exact plugin, not an earlier plugin that shares the driver key", async () => {
    // The helper seeds the plugin that owns the lease. Pin it to `pluginId`
    // through the lease metadata and give it a lower-priority sibling.
    const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease();

    // Rewrite the owner plugin so it DENIES reusable leases through the nested
    // capability declaration.
    await db
      .update(plugins)
      .set({
        manifestJson: {
          id: "acme.reusable-sandbox-provider",
          apiVersion: 1,
          version: "1.0.0",
          displayName: "Reusable Sandbox Provider",
          description: "Owner plugin that denies reusable leases",
          author: "Paperclip",
          categories: ["automation"],
          capabilities: ["environment.drivers.register"],
          entrypoints: { worker: "dist/worker.js" },
          environmentDrivers: [
            {
              driverKey: "fake-plugin",
              kind: "sandbox_provider",
              displayName: "Fake Plugin",
              sandboxCapabilities: { reusableLeases: false },
              configSchema: { type: "object", properties: {} },
            },
          ],
        },
        updatedAt: new Date(),
      } as any)
      .where(eq(plugins.id, pluginId));

    // Install an EARLIER plugin that shares the driver key and grants reusable
    // leases. It is not ready and never acquired this lease. A resolver keyed by
    // driver key alone would read this declaration and grant a capability the
    // owner plugin denied.
    const collidingPluginId = randomUUID();
    await db.insert(plugins).values({
      id: collidingPluginId,
      pluginKey: "acme.colliding-sandbox-provider",
      packageName: "@acme/colliding-sandbox-provider",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.colliding-sandbox-provider",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Colliding Sandbox Provider",
        description: "Earlier plugin that shares the driver key",
        author: "Paperclip",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            supportsReusableLeases: true,
            configSchema: { type: "object", properties: {} },
          },
        ],
      },
      status: "installed",
      installOrder: 0,
      updatedAt: new Date(),
    } as any);

    // The owner worker verifies the reuse verbs and the sync verbs, so every
    // capability is verified. Only the owner's declaration can narrow one.
    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(),
      getWorker: vi.fn((id: string) =>
        id === pluginId
          ? {
              supportedMethods: [
                "environmentResumeLease",
                "environmentReleaseLease",
                "environmentSyncIn",
                "environmentSyncOut",
              ],
            }
          : undefined,
      ),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    expect(reusableLease.metadata?.pluginId).toBe(pluginId);

    const effective = await runtimeWithPlugin.resolveCapabilities({
      environment,
      lease: reusableLease,
    });

    // The runtime read the owner plugin's declaration, so it denies reusable
    // leases even though the earlier plugin grants them.
    expect(effective?.reusableLeases).toBe(false);
    // The owner plugin does not restrict native sync, and its worker verifies
    // the sync verbs, so those stay granted. This proves the resolver read the
    // owner declaration and did not fail every capability closed.
    expect(effective?.nativeSyncIn).toBe(true);
    expect(effective?.nativeSyncOut).toBe(true);
  });

  it("fails every effective capability closed when the pinned plugin id is absent from the registry", async () => {
    // The lease pins a plugin id, but that plugin record is gone. A stale worker
    // entry still advertises every method. The runtime must not read the stale
    // methods; it must fail closed because the exact-plugin identity is gone.
    const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease();
    await db.delete(plugins).where(eq(plugins.id, pluginId));

    const workerManager = {
      isRunning: vi.fn(() => true),
      call: vi.fn(),
      getWorker: vi.fn(() => ({
        supportedMethods: [
          "environmentResumeLease",
          "environmentReleaseLease",
          "environmentExecute",
          "environmentSyncIn",
          "environmentSyncOut",
        ],
      })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const effective = await runtimeWithPlugin.resolveCapabilities({
      environment,
      lease: reusableLease,
    });

    for (const key of SANDBOX_CAPABILITY_KEYS) {
      expect(effective?.[key]).toBe(false);
    }
  });

  it("fails every effective capability closed when the pinned plugin no longer declares this provider key", async () => {
    // The pinned plugin still exists, but it no longer declares a
    // `sandbox_provider` driver with this key (here it changed the driver kind).
    // A running worker still advertises every method. The runtime must fail
    // closed because the exact-plugin declaration is gone.
    const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease();
    await db
      .update(plugins)
      .set({
        manifestJson: {
          id: "acme.reusable-sandbox-provider",
          apiVersion: 1,
          version: "1.0.0",
          displayName: "Reusable Sandbox Provider",
          description: "Owner plugin that no longer declares the provider key",
          author: "Paperclip",
          categories: ["automation"],
          capabilities: ["environment.drivers.register"],
          entrypoints: { worker: "dist/worker.js" },
          environmentDrivers: [
            {
              driverKey: "fake-plugin",
              // The key exists, but the kind is now a plain environment driver,
              // not a sandbox provider. The by-id resolver fails closed.
              kind: "environment_driver",
              displayName: "Fake Plugin",
              configSchema: { type: "object", properties: {} },
            },
          ],
        },
        updatedAt: new Date(),
      } as any)
      .where(eq(plugins.id, pluginId));

    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(),
      getWorker: vi.fn(() => ({
        supportedMethods: [
          "environmentResumeLease",
          "environmentReleaseLease",
          "environmentExecute",
          "environmentSyncIn",
          "environmentSyncOut",
        ],
      })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const effective = await runtimeWithPlugin.resolveCapabilities({
      environment,
      lease: reusableLease,
    });

    for (const key of SANDBOX_CAPABILITY_KEYS) {
      expect(effective?.[key]).toBe(false);
    }
  });

  it("defers to verified worker discovery for a valid pinned plugin that omits sandboxCapabilities", async () => {
    // A valid, identified plugin whose manifest declares no `sandboxCapabilities`
    // and no legacy reuse flag. Its worker verifies the sync verbs. An omitted
    // declaration is NOT an identity failure: the runtime defers to the verified
    // baseline, so native sync stays granted while unverified capabilities stay
    // false.
    const { pluginId, environment, reusableLease } = await seedReusablePluginSandboxLease();
    await db
      .update(plugins)
      .set({
        manifestJson: {
          id: "acme.reusable-sandbox-provider",
          apiVersion: 1,
          version: "1.0.0",
          displayName: "Reusable Sandbox Provider",
          description: "Owner plugin that omits the capability declaration",
          author: "Paperclip",
          categories: ["automation"],
          capabilities: ["environment.drivers.register"],
          entrypoints: { worker: "dist/worker.js" },
          environmentDrivers: [
            {
              driverKey: "fake-plugin",
              kind: "sandbox_provider",
              displayName: "Fake Plugin",
              configSchema: { type: "object", properties: {} },
            },
          ],
        },
        updatedAt: new Date(),
      } as any)
      .where(eq(plugins.id, pluginId));

    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(),
      getWorker: vi.fn((id: string) =>
        id === pluginId
          ? { supportedMethods: ["environmentSyncIn", "environmentSyncOut"] }
          : undefined,
      ),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const effective = await runtimeWithPlugin.resolveCapabilities({
      environment,
      lease: reusableLease,
    });

    // The worker verified the sync verbs and the omitted declaration adds no
    // restriction, so native sync stays granted.
    expect(effective?.nativeSyncIn).toBe(true);
    expect(effective?.nativeSyncOut).toBe(true);
    // The worker did not verify the reuse verbs, so reusable leases stay false.
    expect(effective?.reusableLeases).toBe(false);
  });

  it("releases a sandbox run lease from metadata after the environment config changes", async () => {
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "sandbox",
      name: "Fake Sandbox",
      config: {
        provider: "fake",
        image: "ubuntu:24.04",
        reuseLease: true,
      },
    });

    const acquired = await runtime.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
    });

    await environmentService(db).update(environment.id, {
      driver: "local",
      config: {},
    });

    const released = await runtime.releaseRunLeases(runId);

    expect(released).toHaveLength(1);
    expect(released[0]?.lease.id).toBe(acquired.lease.id);
    expect(released[0]?.lease.status).toBe("released");
  });

  it("does not reuse a sandbox lease owned by a different agent for the same execution workspace", async () => {
    const { companyId, agentId, environment, runId } = await seedEnvironment({
      driver: "plugin",
      name: "Plugin Fake plugin",
      config: {
        pluginKey: "acme.environments",
        driverKey: "fake-plugin",
        driverConfig: {
          template: "base",
        },
      },
    });
    const otherAgentId = randomUUID();
    const executionWorkspaceId = randomUUID();
    const pluginId = randomUUID();
    const projectId = randomUUID();

    await db.insert(projects).values({
      id: projectId,
      companyId,
      name: `Workspace ${projectId.slice(0, 8)}`,
      status: "active",
      createdAt: new Date(),
      updatedAt: new Date(),
    });
    await db.insert(executionWorkspaces).values({
      id: executionWorkspaceId,
      companyId,
      projectId,
      mode: "shared_workspace",
      strategyType: "project_primary",
      name: "Existing workspace",
      status: "active",
      providerType: "local_fs",
      createdAt: new Date(),
      updatedAt: new Date(),
    });

    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "acme.environments",
      packageName: "@acme/paperclip-environments",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.environments",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Acme Environments",
        description: "Test plugin environment driver",
        author: "Acme",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            kind: "sandbox_provider",
            displayName: "Fake Plugin",
            configSchema: { type: "object" },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);

    await environmentService(db).acquireLease({
      companyId,
      environmentId: environment.id,
      executionWorkspaceId,
      heartbeatRunId: runId,
      leasePolicy: "reuse_by_environment",
      provider: "fake-plugin",
      providerLeaseId: "other-agent-lease",
      metadata: {
        agentId,
        provider: "fake-plugin",
        template: "base",
        reuseLease: true,
      },
    });

    const workerManager = {
      isRunning: vi.fn((id: string) => id === pluginId),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "fresh-agent-lease",
            metadata: {
              provider: "fake-plugin",
              template: "base",
              reuseLease: true,
            },
          };
        }
        throw new Error(`Unexpected plugin method: ${method}`);
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });

    const acquired = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      agentId: otherAgentId,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: {
        id: executionWorkspaceId,
        mode: "shared_workspace",
      },
    });

    expect(acquired.lease.providerLeaseId).toBe("fresh-agent-lease");
    expect(workerManager.call).toHaveBeenCalledTimes(1);
    expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.objectContaining({
      agentId: otherAgentId,
      executionWorkspaceId,
    }));
  });

  it.each([undefined, 300_000])("delegates plugin environment leases with acquisition budget %s", async (defaultAcquireTimeoutMs) => {
    const pluginId = randomUUID();
    const expiresAt = new Date(Date.now() + 60_000).toISOString();
    const workerManager = {
      isRunning: vi.fn(() => true),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "plugin-lease-1",
            expiresAt,
            metadata: {
              driver: "local",
              pluginId: "provider-plugin-id",
              pluginKey: "provider.plugin",
              driverKey: "provider-driver",
              executionWorkspaceMode: "provider-mode",
              provider: "test-provider",
              remoteCwd: "/workspace",
            },
          };
        }
        return undefined;
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, {
      pluginWorkerManager: workerManager,
    });
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "plugin",
      name: "Plugin Fake plugin",
      config: {
        pluginKey: "acme.environments",
        driverKey: "fake-plugin",
        driverConfig: {
          template: "base",
        },
      },
    });

    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "acme.environments",
      packageName: "@acme/paperclip-environments",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.environments",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Acme Environments",
        description: "Test plugin environment driver",
        author: "Acme",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            displayName: "Fake plugin",
            defaultAcquireTimeoutMs,
            configSchema: { type: "object" },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);

    const acquired = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
    });

    expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", {
      driverKey: "fake-plugin",
      companyId,
      environmentId: environment.id,
      executionWorkspaceId: undefined,
      executionWorkspaceSettings: null,
      issueId: null,
      config: { template: "base" },
      agentId: undefined,
      adapterType: undefined,
      runId,
      workspaceMode: undefined,
    }, ...(defaultAcquireTimeoutMs === undefined ? [] : [330_000]));
    expect(acquired.lease.providerLeaseId).toBe("plugin-lease-1");
    expect(acquired.lease.expiresAt?.toISOString()).toBe(expiresAt);
    expect(acquired.lease.metadata).toMatchObject({
      driver: "plugin",
      pluginId,
      pluginKey: "acme.environments",
      driverKey: "fake-plugin",
      executionWorkspaceMode: null,
      providerMetadata: {
        driver: "local",
        pluginId: "provider-plugin-id",
        pluginKey: "provider.plugin",
        driverKey: "provider-driver",
        executionWorkspaceMode: "provider-mode",
        provider: "test-provider",
        remoteCwd: "/workspace",
      },
    });

    await environmentService(db).update(environment.id, {
      driver: "local",
      config: {},
    });

    const released = await runtimeWithPlugin.releaseRunLeases(runId);

    expect(released).toHaveLength(1);
    expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentReleaseLease", {
      driverKey: "fake-plugin",
      companyId,
      environmentId: environment.id,
      issueId: null,
      config: {},
      providerLeaseId: "plugin-lease-1",
      leaseMetadata: expect.objectContaining({
        driver: "plugin",
        pluginId,
        providerMetadata: expect.objectContaining({
          driver: "local",
        }),
      }),
    });
    expect(released[0]?.lease.status).toBe("released");
  });

  async function seedPluginDriverEnvironment(pluginId: string) {
    const seeded = await seedEnvironment({
      driver: "plugin",
      name: "Plugin Fake plugin",
      config: {
        pluginKey: "acme.environments",
        driverKey: "fake-plugin",
        driverConfig: { template: "base" },
      },
    });
    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "acme.environments",
      packageName: "@acme/paperclip-environments",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.environments",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Acme Environments",
        description: "Test plugin environment driver",
        author: "Acme",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          { driverKey: "fake-plugin", displayName: "Fake plugin", configSchema: { type: "object" } },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);
    return seeded;
  }

  it("forwards a requested expiry to the acquire RPC and records only a provider-attested expiry", async () => {
    const pluginId = randomUUID();
    let receivedRequestedExpiresAt: unknown;
    const workerManager = {
      isRunning: vi.fn(() => true),
      call: vi.fn(async (_pluginId: string, method: string, params: Record<string, unknown>) => {
        if (method === "environmentAcquireLease") {
          receivedRequestedExpiresAt = params.requestedExpiresAt;
          // The provider grants no expiry.
          return { providerLeaseId: "plugin-lease-no-expiry", metadata: { remoteCwd: "/workspace" } };
        }
        return undefined;
      }),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
    const { companyId, environment, runId } = await seedPluginDriverEnvironment(pluginId);

    const requestedExpiresAt = new Date(Date.now() + 60_000);
    const acquired = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
      requestedExpiresAt,
    });

    // The acquire RPC receives the requested expiry as an ISO 8601 string.
    expect(receivedRequestedExpiresAt).toBe(requestedExpiresAt.toISOString());
    // The provider gave no expiry, so the lease row records no expiry. The runtime
    // never synthesizes the requested deadline as evidence of provider enforcement.
    expect(acquired.lease.expiresAt ?? null).toBeNull();
  });

  it("records the real provider expiry when the provider grants one later than the requested deadline", async () => {
    const pluginId = randomUUID();
    const providerExpiresAt = new Date(Date.now() + 120_000).toISOString();
    const workerManager = {
      isRunning: vi.fn(() => true),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "plugin-lease-later",
            expiresAt: providerExpiresAt,
            metadata: { remoteCwd: "/workspace" },
          };
        }
        return undefined;
      }),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
    const { companyId, environment, runId } = await seedPluginDriverEnvironment(pluginId);

    const acquired = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
      requestedExpiresAt: new Date(Date.now() + 60_000),
    });

    // The lease row records the real provider expiry, not the requested deadline.
    // The caller then verifies the expiry bounds its deadline and fails closed.
    expect(acquired.lease.expiresAt?.toISOString()).toBe(providerExpiresAt);
  });

  it("delegates the full plugin environment lifecycle through the worker manager", async () => {
    const pluginId = randomUUID();
    const workerManager = {
      isRunning: vi.fn(() => true),
      call: vi.fn(async (_pluginId: string, method: string) => {
        if (method === "environmentAcquireLease") {
          return {
            providerLeaseId: "plugin-lease-full",
            metadata: {
              remoteCwd: "/workspace",
            },
          };
        }
        if (method === "environmentResumeLease") {
          return {
            providerLeaseId: "plugin-lease-full",
            metadata: {
              resumed: true,
            },
          };
        }
        if (method === "environmentRealizeWorkspace") {
          return {
            cwd: "/workspace/project",
            metadata: {
              realized: true,
            },
          };
        }
        if (method === "environmentExecute") {
          return {
            exitCode: 0,
            signal: null,
            timedOut: false,
            stdout: "ok\n",
            stderr: "",
            metadata: {
              commandId: "cmd-1",
            },
          };
        }
        return undefined;
      }),
      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
    } as unknown as PluginWorkerManager;
    const runtimeWithPlugin = environmentRuntimeService(db, {
      pluginWorkerManager: workerManager,
    });
    const { companyId, environment, runId } = await seedEnvironment({
      driver: "plugin",
      name: "Plugin Full Lifecycle",
      config: {
        pluginKey: "acme.environments",
        driverKey: "fake-plugin",
        driverConfig: {
          template: "base",
        },
      },
    });

    await db.insert(plugins).values({
      id: pluginId,
      pluginKey: "acme.environments",
      packageName: "@acme/paperclip-environments",
      version: "1.0.0",
      apiVersion: 1,
      categories: ["automation"],
      manifestJson: {
        id: "acme.environments",
        apiVersion: 1,
        version: "1.0.0",
        displayName: "Acme Environments",
        description: "Test plugin environment driver",
        author: "Acme",
        categories: ["automation"],
        capabilities: ["environment.drivers.register"],
        entrypoints: { worker: "dist/worker.js" },
        environmentDrivers: [
          {
            driverKey: "fake-plugin",
            displayName: "Fake plugin",
            configSchema: { type: "object" },
          },
        ],
      },
      status: "ready",
      installOrder: 1,
      updatedAt: new Date(),
    } as any);

    const acquired = await runtimeWithPlugin.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
    });
    const resumed = await runtimeWithPlugin.resumeRunLease({
      environment,
      lease: acquired.lease,
    });
    const realized = await runtimeWithPlugin.realizeWorkspace({
      environment,
      lease: acquired.lease,
      workspace: {
        localPath: "/tmp/project",
        mode: "ephemeral",
      },
    });
    const executed = await runtimeWithPlugin.execute({
      environment,
      lease: acquired.lease,
      command: "echo",
      args: ["ok"],
      cwd: realized.cwd,
      env: { FOO: "bar" },
      stdin: "",
      timeoutMs: 1000,
    });
    const destroyed = await runtimeWithPlugin.destroyRunLease({
      environment,
      lease: acquired.lease,
    });

    expect(resumed).toMatchObject({
      providerLeaseId: "plugin-lease-full",
      metadata: {
        resumed: true,
      },
    });
    expect(realized).toEqual({
      cwd: "/workspace/project",
      metadata: {
        realized: true,
      },
    });
    expect(executed).toMatchObject({
      exitCode: 0,
      timedOut: false,
      stdout: "ok\n",
    });
    expect(destroyed?.status).toBe("failed");
    expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentResumeLease", {
      driverKey: "fake-plugin",
      companyId,
      environmentId: environment.id,
      issueId: null,
      config: { template: "base" },
      providerLeaseId: "plugin-lease-full",
      leaseMetadata: expect.objectContaining({
        driver: "plugin",
        pluginId,
      }),
    });
    expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentRealizeWorkspace", expect.objectContaining({
      driverKey: "fake-plugin",
      companyId,
      environmentId: environment.id,
      config: { template: "base" },
      workspace: {
        localPath: "/tmp/project",
        mode: "ephemeral",
      },
    }));
    expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentExecute", expect.objectContaining({
      driverKey: "fake-plugin",
      companyId,
      environmentId: environment.id,
      command: "echo",
      args: ["ok"],
      cwd: "/workspace/project",
      env: { FOO: "bar" },
    }), 31000);
    expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentDestroyLease", {
      driverKey: "fake-plugin",
      companyId,
      environmentId: environment.id,
      issueId: null,
      config: { template: "base" },
      providerLeaseId: "plugin-lease-full",
      leaseMetadata: expect.objectContaining({
        driver: "plugin",
        pluginId,
      }),
    });
  });

  it("releases with the driver captured on the lease even if the environment driver changes later", async () => {
    const { companyId, environment, runId } = await seedEnvironment();
    const environmentsSvc = environmentService(db);
    const localRelease = vi.fn(async ({ lease, status }: { lease: { id: string }; status: "released" | "expired" | "failed" }) =>
      await environmentsSvc.releaseLease(lease.id, status)
    );
    const sshRelease = vi.fn(async () => {
      throw new Error("ssh release should not be called");
    });
    const runtimeWithSpies = environmentRuntimeService(db, {
      drivers: [
        {
          driver: "local",
          acquireRunLease: async (input) => await environmentsSvc.acquireLease({
            companyId: input.companyId,
            environmentId: input.environment.id,
            executionWorkspaceId: input.executionWorkspaceId,
            issueId: input.issueId,
            heartbeatRunId: input.heartbeatRunId,
            metadata: {
              driver: input.environment.driver,
              executionWorkspaceMode: input.executionWorkspaceMode,
            },
          }),
          releaseRunLease: localRelease,
        },
        {
          driver: "ssh",
          acquireRunLease: async () => {
            throw new Error("ssh acquire should not be called");
          },
          releaseRunLease: sshRelease,
        },
      ],
    });

    const acquired = await runtimeWithSpies.acquireRunLease({
      companyId,
      environment,
      issueId: null,
      heartbeatRunId: runId,
      persistedExecutionWorkspace: null,
    });

    await environmentsSvc.update(environment.id, { driver: "ssh" });

    const released = await runtimeWithSpies.releaseRunLeases(runId);

    expect(released).toHaveLength(1);
    expect(localRelease).toHaveBeenCalledTimes(1);
    expect(sshRelease).not.toHaveBeenCalled();
    expect(acquired.lease.metadata?.driver).toBe("local");
  });

  it("test_release_run_leases_continues_after_first_release_fails", async () => {
    const { companyId, environment, runId } = await seedEnvironment();
    const environmentsSvc = environmentService(db);

    // Seed two active leases for one run. The first driver release throws; the
    // second must still release.
    const failingLease = await environmentsSvc.acquireLease({
      companyId,
      environmentId: environment.id,
      heartbeatRunId: runId,
      provider: "local",
      providerLeaseId: "fail-release",
      metadata: { driver: "local" },
    });
    const healthyLease = await environmentsSvc.acquireLease({
      companyId,
      environmentId: environment.id,
      heartbeatRunId: runId,
      provider: "local",
      providerLeaseId: "healthy-release",
      metadata: { driver: "local" },
    });

    const runtimeWithFailingDriver = environmentRuntimeService(db, {
      drivers: [
        {
          driver: "local",
          acquireRunLease: async () => {
            throw new Error("acquire should not be called");
          },
          releaseRunLease: async ({ lease, status }) => {
            if (lease.providerLeaseId === "fail-release") {
              throw new Error("driver release failed");
            }
            return await environmentsSvc.releaseLease(lease.id, status);
          },
        },
      ],
    });

    const errors: Array<{ leaseId: string; error: unknown }> = [];
    const released = await runtimeWithFailingDriver.releaseRunLeases(
      runId,
      "released",
      (leaseId, error) => errors.push({ leaseId, error }),
    );

    // The healthy lease released even though the first release failed.
    expect(released).toHaveLength(1);
    expect(released[0]?.lease.id).toBe(healthyLease.id);
    expect(released[0]?.lease.status).toBe("released");

    // The failed release reported one lease-specific error.
    expect(errors).toHaveLength(1);
    expect(errors[0]?.leaseId).toBe(failingLease.id);
    expect((errors[0]?.error as Error).message).toBe("driver release failed");

    // The database confirms the isolation. The healthy lease released; the
    // failing lease stayed active.
    const rows = await db.select().from(environmentLeases);
    expect(rows.find((row) => row.id === healthyLease.id)?.status).toBe("released");
    expect(rows.find((row) => row.id === failingLease.id)?.status).toBe("active");
  });
});