PaperClipAI/server/src/services/environment-run-orchestrator.ts

import { RetainedSandboxAbsentError } from "./retained-sandbox-absent-error.js";
import { freshWorkspaceSchema } from "@paperclipai/plugin-sdk";
import { prepareRemoteWorkspace, readRemoteWorkspaceAuthority, remoteWorkspaceRepositoryPath } from "./remote-workspaces.js";
/**
 * Centralized environment run orchestrator.
 *
 * Owns the full environment lifecycle for a heartbeat run:
 *   1. Resolve selected environment
 *   2. Validate environment is active and allowed
 *   3. Acquire or resume lease
 *   4. Realize workspace in the environment
 *   5. Resolve execution target for the adapter
 *   6. Release / retain / fail lease according to policy
 *   7. Record activity and operator-visible status
 *
 * Heartbeat callers delegate to this service instead of inlining
 * environment resolution, lease management, workspace realization,
 * and transport logic.
 */

import { environmentLeases, type Db } from "@paperclipai/db";
import { and, eq } from "drizzle-orm";
import { remoteExecutionHasStopped } from "./remote-execution-termination.js";
import { remoteRecoveryLeaseMismatch, type RemoteRecoveryLeaseMismatchField } from "./native-runtime/remote-recovery-lease-mismatch.js";
import { EnvironmentProviderUnavailableError, readTransientEnvironmentProviderFailure } from "./environment-provider-unavailable-error.js";
import { findReusableSandboxResumeError } from "./reusable-sandbox-resume-error.js";
import { findPluginWorkerUnavailableError } from "./plugin-worker-unavailable-error.js";
import type {
  Environment,
  EnvironmentLease,
  EnvironmentLeasePolicy,
  EnvironmentLeaseStatus,
  ExecutionWorkspace,
  ExecutionWorkspaceConfig,
  IssueExecutionWorkspaceSettings,
} from "@paperclipai/shared";
import {
  resolveRunnerEnvironmentForRun,
  resolveRunnerReconnectGraceMsForRun,
  resolveWarmRunnerIdleTimeoutMsForRun,
} from "./runner-environment-lifecycle.js";
import { environmentService } from "./environments.js";
import {
  environmentRuntimeService,
  buildEnvironmentLeaseContext,
  type EnvironmentRuntimeLeaseRecord,
  type EnvironmentRuntimeService,
  type ProviderResourceDisposition,
} from "./environment-runtime.js";
import { ENVIRONMENT_DRIVER_TRAITS } from "./environment-driver-traits.js";
import {
  resolveEnvironmentExecutionTarget,
  resolveEnvironmentExecutionTransport,
} from "./environment-execution-target.js";
import {
  adapterExecutionTargetToRemoteSpec,
  type AdapterExecutionTarget,
  type AdapterRemoteExecutionSpec,
  type AdapterWorkspaceRealization,
} from "@paperclipai/adapter-utils/execution-target";
import type { DuplexObservabilityRecorder } from "@paperclipai/adapter-utils/duplex-observability";
import { buildWorkspaceRealizationRequest } from "./workspace-realization.js";
import { executionWorkspaceService } from "./execution-workspaces.js";
import { logActivity } from "./activity-log.js";
import { logger } from "../middleware/logger.js";
import { parseObject } from "../adapters/utils.js";
import type { RealizedExecutionWorkspace } from "./workspace-runtime.js";
import type { PluginWorkerManager } from "./plugin-worker-manager.js";

// ---------------------------------------------------------------------------
// Error types
// ---------------------------------------------------------------------------

export type EnvironmentErrorCode =
  | "environment_not_found"
  | "environment_inactive"
  | "unsupported_environment"
  | "unsupported_adapter_environment"
  | "probe_failed"
  | "lease_acquire_failed"
  | "workspace_realization_failed"
  | "transport_resolution_failed"
  | "lease_release_failed"
  | "lease_cleanup_failed";

export class EnvironmentRunError extends Error {
  code: EnvironmentErrorCode;
  environmentId?: string;
  driver?: string;
  provider?: string;
  cause?: unknown;

  constructor(
    code: EnvironmentErrorCode,
    message: string,
    details?: {
      environmentId?: string;
      driver?: string;
      provider?: string;
      cause?: unknown;
    },
  ) {
    super(message);
    this.name = "EnvironmentRunError";
    this.code = code;
    this.environmentId = details?.environmentId;
    this.driver = details?.driver;
    this.provider = details?.provider;
    this.cause = details?.cause;
  }
}

// ---------------------------------------------------------------------------
// Orchestration result types
// ---------------------------------------------------------------------------

export interface EnvironmentAcquisitionResult {
  environment: Environment;
  lease: EnvironmentLease;
  leaseContext: ReturnType<typeof buildEnvironmentLeaseContext>;
  executionTransport: Record<string, unknown> | null;
}

export interface EnvironmentRealizationResult {
  lease: EnvironmentLease;
  workspaceRealization: Record<string, unknown>;
  executionTarget: AdapterExecutionTarget | null;
  remoteExecution: AdapterRemoteExecutionSpec | null;
  persistedExecutionWorkspace: ExecutionWorkspace | null;
}

export interface EnvironmentReleaseResult {
  released: EnvironmentRuntimeLeaseRecord[];
  errors: Array<{ leaseId: string; error: unknown }>;
}

function firstNonEmptyLine(text: string | null | undefined): string | null {
  if (!text) return null;
  for (const rawLine of text.split(/\r?\n/)) {
    const line = rawLine.trim();
    if (line) return line;
  }
  return null;
}

function formatProvisionFailureDetail(result: {
  exitCode: number | null;
  signal?: string | null;
  timedOut: boolean;
  stdout: string;
  stderr: string;
}): string {
  if (result.timedOut) {
    return "provision command timed out";
  }
  const signal = typeof result.signal === "string" && result.signal.trim().length > 0
    ? ` (signal ${result.signal.trim()})`
    : "";
  const detail = firstNonEmptyLine(result.stderr) ?? firstNonEmptyLine(result.stdout);
  const status = `exit code ${result.exitCode ?? "null"}${signal}`;
  return detail ? `${status}: ${detail}` : status;
}

// ---------------------------------------------------------------------------
// Service factory
// ---------------------------------------------------------------------------

export function environmentRunOrchestrator(
  db: Db,
  options: {
    pluginWorkerManager?: PluginWorkerManager;
    environmentRuntime?: EnvironmentRuntimeService;
  } = {},
) {
  const environmentsSvc = environmentService(db);
  const executionWorkspacesSvc = executionWorkspaceService(db);
  const environmentRuntime = options.environmentRuntime ?? environmentRuntimeService(db, {
    pluginWorkerManager: options.pluginWorkerManager,
  });

  /**
   * Resolve the selected environment for a run. The caller passes the concrete
   * selected environment id plus the built-in local fallback id used to lazily
   * ensure the local environment row exists.
   */
  async function resolveEnvironment(input: {
    companyId: string;
    selectedEnvironmentId: string;
    localEnvironmentId: string;
  }): Promise<Environment> {
    const environmentId = input.selectedEnvironmentId || input.localEnvironmentId;

    const environment =
      environmentId === input.localEnvironmentId
        ? await environmentsSvc.ensureLocalEnvironment(input.companyId)
        : await environmentsSvc.getById(environmentId);

    if (!environment) {
      throw new EnvironmentRunError("environment_not_found", `Environment "${environmentId}" not found.`, {
        environmentId,
      });
    }

    if (environment.status !== "active") {
      throw new EnvironmentRunError("environment_inactive", `Environment "${environment.name}" is not active (status: ${environment.status}).`, {
        environmentId: environment.id,
        driver: environment.driver,
      });
    }

    return environment;
  }

  /**
   * Acquire an environment lease for a heartbeat run.
   * Wraps the runtime driver's acquire call with standardized error handling.
   */
  async function acquireLease(input: {
    companyId: string;
    environment: Environment;
    configuredEnvironment?: Environment;
    issueId: string | null;
    agentId: string;
    heartbeatRunId: string;
    persistedExecutionWorkspace: Pick<ExecutionWorkspace, "id" | "mode"> | null;
    executionWorkspaceSettings: IssueExecutionWorkspaceSettings | null;
    runnerTaskProjectIds?: readonly string[];
    adapterType: string | null;
    warmRunnerIdleTimeoutMs?: number;
    runnerReconnectGraceMs?: number;
  }): Promise<EnvironmentRuntimeLeaseRecord> {
    try {
      return await environmentRuntime.acquireRunLease(input);
    } catch (err) {
      // A transient provider refusal (HTTP 5xx/429/timeout) is retried by the
      // setup path with its bounded bootstrap budget. Saved-sandbox resume and
      // stopped-worker refusals keep their own, more specific classifications.
      const transient = findReusableSandboxResumeError(err) || findPluginWorkerUnavailableError(err)
        ? null : readTransientEnvironmentProviderFailure(err);
      throw new EnvironmentRunError(
        "lease_acquire_failed",
        `Failed to acquire lease for environment "${input.environment.name}" (${input.environment.driver}): ${err instanceof Error ? err.message : String(err)}`,
        {
          environmentId: input.environment.id,
          driver: input.environment.driver,
          cause: transient
            ? new EnvironmentProviderUnavailableError({ environmentName: input.environment.name,
              driver: input.environment.driver, evidence: transient, cause: err })
            : err,
        },
      );
    }
  }

  /**
   * Resolve the execution transport for an adapter based on the acquired lease.
   */
  async function resolveTransport(input: {
    companyId: string;
    adapterType: string;
    environment: Environment;
    leaseMetadata: Record<string, unknown> | null;
  }): Promise<Record<string, unknown> | null> {
    try {
      return await resolveEnvironmentExecutionTransport({
        db,
        companyId: input.companyId,
        adapterType: input.adapterType,
        environment: input.environment,
        leaseMetadata: input.leaseMetadata,
      });
    } catch (err) {
      throw new EnvironmentRunError(
        "transport_resolution_failed",
        `Failed to resolve execution transport for "${input.environment.name}": ${err instanceof Error ? err.message : String(err)}`,
        {
          environmentId: input.environment.id,
          driver: input.environment.driver,
          cause: err,
        },
      );
    }
  }

  /**
   * Full acquisition flow: resolve environment, acquire lease, resolve transport.
   * This is the primary entry point for heartbeat run setup.
   */
  async function acquireForRun(input: {
    companyId: string;
    selectedEnvironmentId: string;
    localEnvironmentId: string;
    adapterType: string;
    adapterConfig?: Record<string, unknown>;
    admittedLifecycleMode?: "warm" | "per_turn";
    managedAiCredentialIdentity?: string;
    issueId: string | null;
    heartbeatRunId: string;
    agentId: string;
    persistedExecutionWorkspace: Pick<ExecutionWorkspace, "id" | "mode"> | null;
    executionWorkspaceSettings: IssueExecutionWorkspaceSettings | null;
    runnerTaskProjectIds?: readonly string[];
    /** Existing live runner recovery must use its original active lease, including ephemeral leases. */
    reattachRemoteLease?: { kind?: "sandbox"; leaseId: string; providerLeaseId: string; remoteCwd: string }
      | { kind: "plugin_task"; leaseId: string };
    /** A new controller must stop the previous plugin task before allocating a replacement. */
    recoverPluginTask?: boolean;
  }): Promise<EnvironmentAcquisitionResult> {
    // Step 1: Resolve environment
    const selectedEnvironment = await resolveEnvironment({
      companyId: input.companyId,
      selectedEnvironmentId: input.selectedEnvironmentId,
      localEnvironmentId: input.localEnvironmentId,
    });

    const environment = resolveRunnerEnvironmentForRun(
      selectedEnvironment, input.adapterType, input.adapterConfig, input.admittedLifecycleMode, input.managedAiCredentialIdentity,
    );

    if (input.recoverPluginTask && environment.driver === "plugin" && input.adapterType === "paperclip_runner") {
      const prior = await db.select().from(environmentLeases).where(and(
        eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.heartbeatRunId, input.heartbeatRunId),
      ));
      if (prior.some(lease => lease.metadata?.runnerTaskLease !== true || lease.environmentId !== environment.id ||
          lease.issueId !== input.issueId || lease.executionWorkspaceId !== (input.persistedExecutionWorkspace?.id ?? null) ||
          lease.metadata?.agentId !== input.agentId)) throw new Error("native_plugin_recovery_lease_mismatch");
      const errors: unknown[] = [];
      await environmentRuntime.releaseRunLeases(input.heartbeatRunId, "released", (_id, error) => { errors.push(error); });
      if (errors.length || (prior.length && !await remoteExecutionHasStopped(db, input.companyId, input.heartbeatRunId))) {
        throw new Error("native_plugin_recovery_termination_unconfirmed", { cause: errors[0] });
      }
    }

    // Step 2: Acquire lease
    const warmRunnerIdleTimeoutMs = resolveWarmRunnerIdleTimeoutMsForRun(
      selectedEnvironment, input.adapterType, input.adapterConfig, input.admittedLifecycleMode,
    );
    const runnerReconnectGraceMs = input.adapterType
      ? resolveRunnerReconnectGraceMsForRun(selectedEnvironment, input.adapterType)
      : undefined;
    const acquisitionInput = {
      companyId: input.companyId,
      environment,
      configuredEnvironment: selectedEnvironment,
      issueId: input.issueId,
      agentId: input.agentId,
      heartbeatRunId: input.heartbeatRunId,
      persistedExecutionWorkspace: input.persistedExecutionWorkspace,
      executionWorkspaceSettings: input.executionWorkspaceSettings,
      runnerTaskProjectIds: input.runnerTaskProjectIds,
      adapterType: input.adapterType ?? null,
      warmRunnerIdleTimeoutMs,
      runnerReconnectGraceMs,
    };
    let leaseRecord: EnvironmentRuntimeLeaseRecord;
    if (input.reattachRemoteLease) {
      // Reattachment keeps the original lease, including ephemeral leases.
      // Never reacquire: acquisition can replace a non-reusable sandbox.
      const expected = input.reattachRemoteLease;
      const lease = await environmentsSvc.getLeaseById(expected.leaseId);
      const mismatch: RemoteRecoveryLeaseMismatchField | null = !lease ? "lease"
        : lease.companyId !== input.companyId || lease.environmentId !== environment.id ||
          lease.heartbeatRunId !== input.heartbeatRunId || lease.issueId !== input.issueId ||
          lease.executionWorkspaceId !== (input.persistedExecutionWorkspace?.id ?? null) ||
          lease.metadata?.agentId !== input.agentId ? "lease_scope"
        : lease.status !== "active" || lease.releasedAt !== null || lease.cleanupStatus !== null ? "lease_status"
        : lease.expiresAt !== null && new Date(lease.expiresAt).getTime() <= Date.now() ? "lease_expired"
        : expected.kind === "plugin_task"
          ? environment.driver !== "plugin" ? "environment_driver"
            : input.adapterType !== "paperclip_runner" ? "adapter_type"
            : lease.metadata?.runnerTaskLease !== true || lease.metadata?.runnerTaskDispatched !== true
              || lease.metadata?.runnerTaskExecutionStopped === true ? "plugin_task" : null
        : environment.driver !== "sandbox" ? "environment_driver"
        : lease.provider !== environment.config.provider ? "provider"
        : lease.providerLeaseId !== expected.providerLeaseId ? "provider_lease_id"
        : lease.metadata?.remoteCwd !== expected.remoteCwd ? "remote_cwd" : null;
      if (mismatch || !lease) {
        throw remoteRecoveryLeaseMismatch("environment_acquire", mismatch ?? "lease");
      }
      if (expected.kind !== "plugin_task") {
        // The provider may have archived this retained sandbox while its
        // controller was offline. Ready the exact lease under its lifecycle
        // budget before short preparation commands (such as project-auth
        // checks) can run. This API cannot acquire a replacement.
        const resumed = await environmentRuntime.resumeRunLease({
          environment, lease, requestedCwd: expected.remoteCwd,
          ...(warmRunnerIdleTimeoutMs !== undefined ? { warmRunnerIdleTimeoutMs } : {}),
          ...(runnerReconnectGraceMs !== undefined ? { runnerReconnectGraceMs } : {}),
        }).catch((error: unknown) => {
          // Confirmed provider absence is a named lease mismatch; other
          // failures (for example a transient refusal) keep their own error.
          if (error instanceof RetainedSandboxAbsentError) throw remoteRecoveryLeaseMismatch("environment_resume", "lease_expired");
          throw error;
        });
        if (!resumed) throw remoteRecoveryLeaseMismatch("environment_resume", "lease");
        if (resumed.providerLeaseId !== expected.providerLeaseId) {
          throw remoteRecoveryLeaseMismatch("environment_resume", "provider_lease_id");
        }
        if (resumed.metadata?.expired === true) throw remoteRecoveryLeaseMismatch("environment_resume", "lease_expired");
        // Resume confirms allocation readiness. Its optional metadata may
        // describe the allocation root; keep the validated lease's workspace.
      }
      leaseRecord = { environment, lease, leaseContext: buildEnvironmentLeaseContext(input) };
    } else {
      leaseRecord = await acquireLease(acquisitionInput);
    }

    // Step 3: Log lease acquisition activity
    await logActivity(db, {
      companyId: input.companyId,
      actorType: "agent",
      actorId: input.agentId,
      agentId: input.agentId,
      runId: input.heartbeatRunId,
      action: "environment.lease_acquired",
      entityType: "environment_lease",
      entityId: leaseRecord.lease.id,
      issueId: input.issueId,
      details: {
        environmentId: environment.id,
        driver: environment.driver,
        leasePolicy: leaseRecord.lease.leasePolicy,
        provider: leaseRecord.lease.provider,
        executionWorkspaceId: leaseRecord.leaseContext.executionWorkspaceId,
        issueId: input.issueId,
        networkEgress: input.executionWorkspaceSettings?.networkEgress ?? null,
      },
    });

    // Step 4: Resolve execution transport
    const executionTransport = environment.driver === "plugin" && input.adapterType === "paperclip_runner" ? null : await resolveTransport({
      companyId: input.companyId,
      adapterType: input.adapterType,
      environment,
      leaseMetadata: leaseRecord.lease.metadata,
    });

    return {
      environment,
      lease: leaseRecord.lease,
      leaseContext: leaseRecord.leaseContext,
      executionTransport,
    };
  }

  /**
   * Realize workspace in the environment and resolve the execution target.
   *
   * After lease acquisition, this method:
   *   1. Builds a workspace realization request
   *   2. Calls the environment runtime driver to realize the workspace
   *   3. Persists realization metadata on the lease and execution workspace
   *   4. Resolves the adapter execution target (local/ssh/sandbox)
   *
   * Returns the updated lease, realization metadata, and the execution
   * target spec that the adapter needs to run.
   */
  async function realizeForRun(input: {
    environment: Environment;
    lease: EnvironmentLease;
    adapterType: string;
    companyId: string;
    issueId: string | null;
    heartbeatRunId: string;
    executionWorkspace: RealizedExecutionWorkspace;
    effectiveExecutionWorkspaceMode: string | null;
    persistedExecutionWorkspace: ExecutionWorkspace | null;
    runnerTaskProjectIds?: readonly string[];
    assertOwned?: () => Promise<void>;
    /**
     * The host duplex observability recorder for this run. The orchestrator threads
     * it to `resolveEnvironmentExecutionTarget`, which stamps it on the sandbox
     * target. Absent keeps the safe no-op default in the bridge.
     */
    duplexObservabilityRecorder?: DuplexObservabilityRecorder | null;
  }): Promise<EnvironmentRealizationResult> {
    const {
      environment,
      adapterType,
      companyId,
      issueId,
      heartbeatRunId,
      executionWorkspace,
      effectiveExecutionWorkspaceMode,
    } = input;
    let { lease, persistedExecutionWorkspace } = input;
    const remoteAuthority = readRemoteWorkspaceAuthority(persistedExecutionWorkspace?.metadata);
    if (remoteAuthority && persistedExecutionWorkspace) {
      if (adapterType !== "paperclip_runner" || environment.driver !== "sandbox") {
        throw new EnvironmentRunError("unsupported_adapter_environment", "Remote workspace storage requires a qualified native sandbox");
      }
      const target = await resolveEnvironmentExecutionTarget({
        db, companyId, adapterType, environment, leaseId: lease.id, leaseMetadata: lease.metadata,
        lease, environmentRuntime, duplexObservabilityRecorder: input.duplexObservabilityRecorder ?? null,
      });
      if (!target || target.kind !== "remote" || target.transport !== "sandbox" || !target.runner || target.runnerTask) {
        throw new EnvironmentRunError("transport_resolution_failed", "Remote workspace storage requires a sandbox file transport");
      }
      const prepared = await prepareRemoteWorkspace({
        db, workspace: persistedExecutionWorkspace, runId: heartbeatRunId, lease, target,
        source: remoteAuthority.source,
        assertOwned: input.assertOwned,
        provisionCommand: persistedExecutionWorkspace.config?.provisionCommand,
        runtimeProvisionCommand: persistedExecutionWorkspace.config?.runtimeProvisionCommand,
      });
      lease = prepared.lease;
      const workspaceRealization = {
        version: 1, mode: "in_place", authoritativeRoot: prepared.remoteRoot,
        pathAliases: [], outboundRestorePaths: [], provider: lease.provider,
        environmentId: environment.id, leaseId: lease.id, providerLeaseId: lease.providerLeaseId,
        additional: (remoteAuthority.source.repositories ?? []).filter(repository => repository.readOnly)
          .map(repository => ({ path: remoteWorkspaceRepositoryPath(prepared.remoteRoot, repository),
            projectId: repository.projectId ?? null, projectWorkspaceId: repository.projectWorkspaceId ?? null,
            repoUrl: repository.repoUrl, repoRef: repository.ref })),
      };
      const updatedLease = await environmentsSvc.updateLeaseMetadata(lease.id, {
        ...lease.metadata, remoteCwd: prepared.remoteRoot, workspaceRealization,
      });
      if (!updatedLease) throw new Error("Remote workspace lease disappeared during preparation");
      lease = updatedLease;
      persistedExecutionWorkspace = await executionWorkspacesSvc.getStoredById(companyId, prepared.workspace.id);
      if (!persistedExecutionWorkspace) throw new Error("Remote workspace disappeared during preparation");
      input.executionWorkspace.cwd = prepared.remoteRoot;
      const executionTarget: AdapterExecutionTarget = {
        ...target, remoteCwd: prepared.remoteRoot,
        workspaceRealization: { mode: "in_place", authoritativeRoot: prepared.remoteRoot,
          pathAliases: [], outboundRestorePaths: [] },
      };
      return { lease, workspaceRealization, executionTarget,
        remoteExecution: adapterExecutionTargetToRemoteSpec(executionTarget), persistedExecutionWorkspace };
    }

    if (executionWorkspace.providerBinding) {
      const binding = executionWorkspace.providerBinding;
      if (environment.driver !== "plugin" || adapterType !== "paperclip_runner" || lease.metadata?.pluginId !== binding.pluginId || !await environmentRuntime.supportsTasks({ lease })) {
        throw new EnvironmentRunError("workspace_realization_failed", "The execution plugin cannot access this managed workspace.", { environmentId: environment.id });
      }
      const updated = await environmentsSvc.updateLeaseMetadata(lease.id, { ...lease.metadata, workspaceProvider: binding });
      if (!updated) throw new Error("Workspace execution lease disappeared");
      lease = updated;
    }
    const freshWorkspace = freshWorkspaceSchema.safeParse(persistedExecutionWorkspace?.metadata?.workspaceProviderFresh ??
      persistedExecutionWorkspace?.metadata?.runnerTaskFreshWorkspace);
    if (environment.driver === "plugin" && freshWorkspace.success) {
      const updated = await environmentsSvc.updateLeaseMetadata(lease.id, { ...lease.metadata, freshWorkspace: freshWorkspace.data });
      if (!updated) throw new Error("Workspace execution lease disappeared");
      lease = updated;
    }
    // Step 1: Build workspace realization request
    const workspaceRealizationRequest = buildWorkspaceRealizationRequest({
      adapterType,
      companyId,
      environmentId: environment.id,
      executionWorkspaceId: persistedExecutionWorkspace?.id ?? null,
      issueId,
      heartbeatRunId,
      requestedMode: persistedExecutionWorkspace?.mode ?? effectiveExecutionWorkspaceMode,
      workspace: executionWorkspace,
      workspaceConfig: persistedExecutionWorkspace?.config ?? null,
    });

    // Step 2: Realize workspace in the environment via the runtime driver
    let workspaceRealization: Record<string, unknown> = {};
    let realizedWorkspaceCwd: string | null = null;
    const pluginTaskWorkspace = environment.driver === "plugin" && adapterType === "paperclip_runner";
    if (pluginTaskWorkspace && !await environmentRuntime.supportsTasks({ lease })) {
      throw new EnvironmentRunError("unsupported_adapter_environment", "Plugin environment does not provide native Runner task execution");
    }
    if (ENVIRONMENT_DRIVER_TRAITS[environment.driver].realizesWorkspace) {
      try {
        const remoteCwd =
          typeof lease.metadata?.remoteCwd === "string" && lease.metadata.remoteCwd.trim().length > 0
            ? lease.metadata.remoteCwd
            : undefined;
        const workspaceRealizationResult = await environmentRuntime.realizeWorkspace({
          environment,
          lease,
          workspace: {
            localPath: executionWorkspace.cwd,
            remotePath: remoteCwd,
            mode: persistedExecutionWorkspace?.mode ?? effectiveExecutionWorkspaceMode ?? undefined,
            metadata: {
              workspaceRealizationRequest,
            },
          },
        });
        realizedWorkspaceCwd =
          typeof workspaceRealizationResult.cwd === "string" && workspaceRealizationResult.cwd.trim().length > 0
            ? workspaceRealizationResult.cwd.trim()
            : null;
        workspaceRealization = parseObject(workspaceRealizationResult.metadata?.workspaceRealization);
      } catch (err) {
        throw new EnvironmentRunError(
          "workspace_realization_failed",
          `Failed to realize workspace for environment "${environment.name}" (${environment.driver}): ${err instanceof Error ? err.message : String(err)}`,
          {
            environmentId: environment.id,
            driver: environment.driver,
            cause: err,
          },
        );
      }
    }

    const provisionCommand = workspaceRealizationRequest.runtimeOverlay.provisionCommand?.trim() ?? "";
    const realizedCwd =
      realizedWorkspaceCwd ??
      (typeof lease.metadata?.remoteCwd === "string" && lease.metadata.remoteCwd.trim().length > 0
        ? lease.metadata.remoteCwd.trim()
        : executionWorkspace.cwd);
    // The host `provisionCommand` runs on the host worktree during the
    // `workspace_provision` step, before the run reaches the environment.
    // A `sandbox`-driver environment does not receive the repo tree here. The
    // sandbox driver `realizeWorkspace` step only creates the remote folder.
    // The adapter uploads the provisioned tree later, in its `stage.sync` step.
    // So the host command must not run inside the still-empty sandbox; it fails
    // there (exit 127). Skip the step for `sandbox`, and keep the existing skip
    // for `local`. Keep the step for `ssh`, which runs the command on the
    // remote host that shares the workspace path.
    const driverSkipsHostProvision =
      environment.driver === "local" || environment.driver === "sandbox" || pluginTaskWorkspace;
    if (provisionCommand && environment.driver === "sandbox") {
      logger.info(
        {
          environmentId: environment.id,
          driver: environment.driver,
          issueId,
          heartbeatRunId,
        },
        "Skip host provisionCommand for sandbox-driver environment; the adapter stage.sync step delivers the provisioned tree",
      );
    }
    if (provisionCommand && !executionWorkspace.providerBinding && !driverSkipsHostProvision) {
      try {
        const provisionResult = await environmentRuntime.execute({
          environment,
          lease,
          command: "bash",
          args: ["-lc", provisionCommand],
          cwd: realizedCwd,
          env: {
            SHELL: "/bin/bash",
          },
          timeoutMs: 300_000,
          // The provision command runs before the run opens its trace root, so it
          // carries no run parent. A sandbox provider that opens a persistent
          // session on the first command must not open the session here, or the
          // session-setup span loses its parent and the span backend drops it.
          // Bypass the session for this command; the session opens on the first
          // in-run command instead, whose setup span parents to the run trace.
          bypassSession: true,
        });
        if (provisionResult.exitCode !== 0 || provisionResult.timedOut) {
          throw new Error(formatProvisionFailureDetail(provisionResult));
        }
      } catch (err) {
        throw new EnvironmentRunError(
          "workspace_realization_failed",
          `Failed to provision workspace for environment "${environment.name}" (${environment.driver}): ${err instanceof Error ? err.message : String(err)}`,
          {
            environmentId: environment.id,
            driver: environment.driver,
            cause: err,
          },
        );
      }
    }

    // Step 3: Persist realization metadata on lease and execution workspace
    if (Object.keys(workspaceRealization).length > 0) {
      const nextLeaseMetadata = {
        ...(lease.metadata ?? {}),
        workspaceRealization,
      };
      const updatedLease = await environmentsSvc.updateLeaseMetadata(lease.id, nextLeaseMetadata);
      if (updatedLease) {
        lease = updatedLease;
      }
      if (persistedExecutionWorkspace) {
        const updatedEw = await executionWorkspacesSvc.update(persistedExecutionWorkspace.id, {
          metadata: {
            ...(persistedExecutionWorkspace.metadata ?? {}),
            workspaceRealizationRequest,
            workspaceRealization,
          },
        });
        if (updatedEw) {
          persistedExecutionWorkspace = updatedEw;
        }
      }
    }

    // Step 4: Resolve execution target for the adapter
    let executionTarget: AdapterExecutionTarget | null;
    try {
      executionTarget = await resolveEnvironmentExecutionTarget({
        db,
        companyId,
        adapterType,
        environment,
        leaseId: lease.id,
        leaseMetadata: (lease.metadata as Record<string, unknown> | null) ?? null,
        lease,
        environmentRuntime,
        duplexObservabilityRecorder: input.duplexObservabilityRecorder ?? null,
      });
      // The native plugin task bridge owns guest paths. Never fall back to host execution.
      if (executionWorkspace.providerBinding && (executionTarget?.kind !== "remote" ||
          !("runnerTask" in executionTarget) || typeof executionTarget.runnerTask !== "function")) {
        throw new Error("Managed workspace execution requires the native plugin task bridge");
      }
      const realizationMode = pluginTaskWorkspace || workspaceRealization.mode === "in_place" ? "in_place" : "copy";
      const authoritativeRoot =
        typeof workspaceRealization.authoritativeRoot === "string" && workspaceRealization.authoritativeRoot.trim().length > 0
          ? workspaceRealization.authoritativeRoot.trim()
          : realizedCwd;
      const workspaceTargetMetadata: AdapterWorkspaceRealization = {
        mode: realizationMode,
        authoritativeRoot,
        pathAliases: Array.isArray(workspaceRealization.pathAliases)
          ? workspaceRealization.pathAliases.filter(
              (entry): entry is { path: string; target: string } =>
                typeof entry === "object" && entry !== null &&
                typeof (entry as { path?: unknown }).path === "string" &&
                typeof (entry as { target?: unknown }).target === "string",
            )
          : [],
        outboundRestorePaths: Array.isArray(workspaceRealization.outboundRestorePaths)
          ? workspaceRealization.outboundRestorePaths.filter((entry): entry is string => typeof entry === "string")
          : [],
      };
      if (executionTarget) {
        executionTarget = {
          ...executionTarget,
          ...(executionTarget.kind === "remote" && executionTarget.transport === "sandbox" && executionTarget.runnerTask
            ? { runnerTaskProjectIds: [...new Set([executionWorkspace.projectId,
                ...input.runnerTaskProjectIds ?? [],
                ...(executionWorkspace.additionalWorkspaces ?? []).map(workspace => workspace.projectId)]
                .filter((id): id is string => Boolean(id)))] } : {}),
          ...(executionTarget.kind === "remote" && realizationMode === "in_place"
            ? { remoteCwd: authoritativeRoot }
            : {}),
          workspaceRealization: workspaceTargetMetadata,
        } as AdapterExecutionTarget;
      }
    } catch (err) {
      throw new EnvironmentRunError(
        "transport_resolution_failed",
        `Failed to resolve execution target for "${environment.name}": ${err instanceof Error ? err.message : String(err)}`,
        {
          environmentId: environment.id,
          driver: environment.driver,
          cause: err,
        },
      );
    }

    return {
      lease,
      workspaceRealization,
      executionTarget,
      remoteExecution: adapterExecutionTargetToRemoteSpec(executionTarget),
      persistedExecutionWorkspace,
    };
  }

  /**
   * Release all active leases for a heartbeat run.
   * Tracks cleanup status per lease. Errors during individual lease release
   * are captured but do not prevent other leases from being released.
   * The original run failure (if any) is never hidden by cleanup errors.
   */
  async function releaseForRun(input: {
    heartbeatRunId: string;
    companyId: string;
    agentId: string;
    status?: Extract<EnvironmentLeaseStatus, "released" | "expired" | "failed">;
    failureReason?: string;
    /** Explicit Stop during adapter startup; never used for ordinary cleanup. */
    cancelActiveWork?: boolean;

    respectCleanupDeadlines?: boolean;
    /** Explicit paperclip_runner resource lifecycle. Omitted for legacy adapters. */
    providerResourceDisposition?: ProviderResourceDisposition;
    nativeLifecycleTelemetry?: {
      provider: string;
      harness: string;
      lifecycleMode: "per_turn" | "warm";
      sandboxResource:
        | "keep_running"
        | "stop_and_reuse"
        | "destroy_after_turn";
    };
  }): Promise<EnvironmentReleaseResult> {
    const status = input.status ?? "released";
    const result: EnvironmentReleaseResult = { released: [], errors: [] };

    let releasedLeases: EnvironmentRuntimeLeaseRecord[];
    try {
      const releaseArgs: Parameters<typeof environmentRuntime.releaseRunLeases> = [
        input.heartbeatRunId,
        status,
        (leaseId, error) => result.errors.push({ leaseId, error }),
        input.providerResourceDisposition,
      ];
      if (input.cancelActiveWork) releaseArgs[4] = true;
      if (input.respectCleanupDeadlines) releaseArgs[5] = { respectCleanupDeadlines: true };
      releasedLeases = await environmentRuntime.releaseRunLeases(...releaseArgs);
    } catch (err) {
      result.errors.push({ leaseId: "*", error: err });
      return result;
    }

    for (const released of releasedLeases) {
      try {
        await logActivity(db, {
          companyId: input.companyId,
          actorType: "agent",
          actorId: input.agentId,
          agentId: input.agentId,
          runId: input.heartbeatRunId,
          action: "environment.lease_released",
          entityType: "environment_lease",
          entityId: released.lease.id,
          issueId: released.lease.issueId,
          details: {
            environmentId: released.lease.environmentId,
            driver: released.environment.driver,
            leasePolicy: released.lease.leasePolicy,
            provider: released.lease.provider,
            executionWorkspaceId: released.lease.executionWorkspaceId,
            issueId: released.lease.issueId,
            status: released.lease.status,
            cleanupStatus: released.lease.cleanupStatus,
            failureReason: input.failureReason ?? released.lease.failureReason,
            providerResourceDisposition:
              input.providerResourceDisposition ?? "legacy_default",
          },
        });
      } catch {
        // Activity logging failure should not block lease release
      }
      result.released.push(released);
    }

    return result;
  }

  return {
    resolveEnvironment,
    acquireLease,
    resolveTransport,
    acquireForRun,
    realizeForRun,
    releaseForRun,

    // Expose the underlying runtime for cases that need direct driver access
    runtime: environmentRuntime,
  };
}

export type EnvironmentRunOrchestrator = ReturnType<typeof environmentRunOrchestrator>;