PaperClipAI/server/src/services/bundled-plugins.ts

import path from "node:path";
import fs from "node:fs";
import { isDeepStrictEqual } from "node:util";
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
import { assertDistributionManifestCapabilities, readDistributionPluginCatalog, type DistributionPlugin } from "./distribution-plugin-catalog.js";

/**
 * Bundled plugin auto-provisioning.
 *
 * Managed-cloud instances receive a `plugins.autoInstall` key list through
 * `PAPERCLIP_MANAGED_CONFIG` (parsed fail-closed at startup — see
 * `managed-config.ts`). Each key maps to a plugin bundled into
 * the release image under the bundled catalog root. Nobody "installs" on a
 * managed instance: the control plane provisions, tenants use.
 *
 * Two distinct failure postures, deliberately split:
 *
 * 1. **Resolution (this file, `resolveBundledPluginInstalls`) fails to
 *    start.** An unknown key or a path that escapes the bundled catalog
 *    root is a configuration/security violation — a positive allowlist,
 *    not a lookup. Throwing here happens
 *    synchronously inside `createApp`, before the server listens, so a bad
 *    document refuses to start rather than silently widening what code can
 *    be loaded into the host.
 *
 * 2. **Installation (`ensureBundledPlugins`) is fail-safe.**
 *    Missing bundle on disk, install error, load error: caught, logged,
 *    and swallowed per entry so the server ALWAYS finishes booting. A
 *    degraded boot (one provider unavailable) is strictly preferable to a
 *    crash loop across a fleet.
 *
 * Removal of a key from `autoInstall` stops future installs but never
 * auto-uninstalls: there is intentionally no uninstall
 * path anywhere in this module.
 */

/** Default location of the bundled plugin catalog inside the release image. */
export const DEFAULT_BUNDLED_CATALOG_ROOT = "/app/packages/plugins";

/**
 * Env var that relocates the bundled catalog root (dev images, tests).
 */
export const BUNDLED_CATALOG_ROOT_ENV_VAR = "PAPERCLIP_BUNDLED_PLUGIN_ROOT";

export interface BundledPluginCatalogEntry {
  /** Key the managed config's `plugins.autoInstall` list uses. */
  key: string;
  /** Manifest id / registry `pluginKey` the bundle installs as. */
  pluginKey: string;
  /** Bundle location relative to the bundled catalog root. */
  relativePath: string;
  /**
   * Legacy absolute-path override honored for compatibility (the kubernetes
   * bundle predates the catalog). Overrides are still subject to catalog
   * containment when enforcement is on.
   */
  pathOverrideEnvVar?: string;
}

/**
 * The positive allowlist of plugins the control plane may auto-provision.
 * Keys outside this table can never be installed through this path,
 * regardless of what the managed config document says.
 */
export const BUNDLED_PLUGIN_CATALOG: readonly BundledPluginCatalogEntry[] = [
  {
    key: "createos",
    pluginKey: "paperclip.createos-sandbox-provider",
    relativePath: "sandbox-providers/createos",
  },
  {
    key: "cloudflare",
    pluginKey: "paperclip.cloudflare-sandbox-provider",
    relativePath: "sandbox-providers/cloudflare",
  },
  {
    key: "daytona",
    pluginKey: "paperclip.daytona-sandbox-provider",
    relativePath: "sandbox-providers/daytona",
  },
  {
    key: "e2b",
    pluginKey: "paperclip.e2b-sandbox-provider",
    relativePath: "sandbox-providers/e2b",
  },
  {
    key: "exe-dev",
    pluginKey: "paperclip.exe-dev-sandbox-provider",
    relativePath: "sandbox-providers/exe-dev",
  },
  {
    key: "kubernetes",
    pluginKey: "paperclip.kubernetes-sandbox-provider",
    relativePath: "sandbox-providers/kubernetes",
    pathOverrideEnvVar: "PAPERCLIP_KUBERNETES_PLUGIN_PATH",
  },
  {
    key: "modal",
    pluginKey: "paperclip.modal-sandbox-provider",
    relativePath: "sandbox-providers/modal",
  },
  {
    key: "novita",
    pluginKey: "paperclip.novita-sandbox-provider",
    relativePath: "sandbox-providers/novita",
  },
];

/**
 * Keys ensured on a self-hosted instance (no managed config present).
 * Exactly the pre-refactor behavior: the kubernetes sandbox provider is
 * auto-installed when its bundle is present, nothing else.
 */
export const SELF_HOSTED_AUTO_INSTALL_KEYS: readonly string[] = ["kubernetes"];

export function resolveBundledCatalogRoot(
  env: Record<string, string | undefined>,
): string {
  const override = env[BUNDLED_CATALOG_ROOT_ENV_VAR]?.trim();
  return override ? override : DEFAULT_BUNDLED_CATALOG_ROOT;
}

export interface ResolvedBundledPlugin {
  key: string;
  pluginKey: string;
  /** Absolute path handed to `loader.installPlugin({ localPath })`. */
  localPath: string;
  /** Image-owned entries require exact manifest identity/version matching. */
  distribution?: DistributionPlugin;
}

/**
 * Canonicalize a path for containment comparison. Symlinks are resolved
 * when the path exists so a link inside the catalog cannot point install
 * resolution at a directory outside it; nonexistent paths fall back to a
 * lexical resolve (`..` segments still collapse).
 */
function canonicalize(p: string): string {
  const resolved = path.resolve(p);
  try {
    return fs.realpathSync(resolved);
  } catch {
    return resolved;
  }
}

function isInsideRoot(candidate: string, root: string): boolean {
  const rel = path.relative(root, candidate);
  return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
}

/**
 * Resolve auto-install keys to concrete bundle paths.
 *
 * Throws — and the instance must refuse to start — when a key is not in
 * the bundled catalog, or when `enforceCatalogRoot` is set and the
 * resolved path escapes the catalog root. Callers pass
 * `enforceCatalogRoot: true` for managed (control-plane-driven) key lists
 * and `false` for the self-hosted built-in list, where the legacy
 * kubernetes path override may point anywhere (unchanged behavior).
 */
export function resolveBundledPluginInstalls(
  keys: readonly string[],
  opts: {
    catalogRoot: string;
    env: Record<string, string | undefined>;
    enforceCatalogRoot: boolean;
    distributionPlugins?: readonly DistributionPlugin[];
  },
): ResolvedBundledPlugin[] {
  const resolved: ResolvedBundledPlugin[] = [];
  const seen = new Set<string>();
  const canonicalRoot = canonicalize(opts.catalogRoot);
  const distributionPlugins = opts.distributionPlugins ?? readDistributionPluginCatalog(opts.catalogRoot, BUNDLED_PLUGIN_CATALOG);
  for (const key of keys) {
    if (seen.has(key)) continue;
    seen.add(key);
    const entry = BUNDLED_PLUGIN_CATALOG.find((candidate) => candidate.key === key);
    if (!entry) {
      const distribution = distributionPlugins.find((candidate) => candidate.key === key);
      if (distribution) {
        resolved.push({ key, pluginKey: distribution.pluginKey, localPath: distribution.localPath, distribution });
        continue;
      }
      const known = BUNDLED_PLUGIN_CATALOG.map((candidate) => candidate.key).join(", ");
      throw new Error(
        `bundled plugin auto-install key "${key}" is not in the bundled catalog (known keys: ${known}); refusing to start`,
      );
    }
    const override = entry.pathOverrideEnvVar
      ? opts.env[entry.pathOverrideEnvVar]?.trim()
      : undefined;
    const localPath = override
      ? path.resolve(override)
      : path.resolve(opts.catalogRoot, entry.relativePath);
    if (opts.enforceCatalogRoot && !isInsideRoot(canonicalize(localPath), canonicalRoot)) {
      throw new Error(
        `bundled plugin "${key}" resolves to "${localPath}", outside the bundled catalog root "${opts.catalogRoot}"; refusing to start`,
      );
    }
    resolved.push({ key: entry.key, pluginKey: entry.pluginKey, localPath });
  }
  return resolved;
}

interface RegistryPluginRow {
  id: string;
  pluginKey: string;
  status: string;
  version: string;
  manifestJson: PaperclipPluginManifestV1;
  packagePath?: string | null;
  lastError?: string | null;
}

export interface BundledPluginProvisionerDeps {
  registry: {
    getByKey(pluginKey: string): Promise<RegistryPluginRow | null>;
    update(
      id: string,
      data: { version?: string; manifest?: PaperclipPluginManifestV1; packagePath?: string; status?: "upgrade_pending" },
    ): Promise<unknown>;
    updateStatus(id: string, input: { status: "ready"; lastError: string | null }): Promise<unknown>;
  };
  loader: {
    installPlugin(options: { localPath: string }): Promise<{
      manifest: { id: string } | null;
    }>;
    loadManifest(packagePath: string): Promise<PaperclipPluginManifestV1 | null>;
  };
  lifecycle: {
    load(pluginId: string): Promise<unknown>;
  };
  logger: {
    info(obj: unknown, msg?: string): void;
    warn(obj: unknown, msg?: string): void;
    error(obj: unknown, msg?: string): void;
  };
  /** Overridable for tests; defaults to checking `dist/manifest.js`. */
  bundleManifestExists?: (localPath: string) => boolean;
}

function defaultBundleManifestExists(localPath: string): boolean {
  return fs.existsSync(path.join(localPath, "dist", "manifest.js"));
}

/**
 * Reconcile a present bundled plugin's persisted manifest with the shipped
 * bundle. The bundle is part of the release image, so its manifest is the
 * source of truth. When the bundle declares a version that differs from the
 * persisted version, update the stored manifest and version. Distribution
 * entries also adopt the image's package path, including same-version installs.
 * This propagates
 * a manifest change (for example a new driver capability) to an existing
 * install that the auto-install path skips.
 *
 * The step is fail-safe. A missing bundle, a manifest read error, or a
 * database error is caught, logged, and swallowed, so boot always completes.
 * The step updates only the stored manifest row; it never restarts the worker.
 */
async function reconcileBundledPluginManifest(
  existing: RegistryPluginRow,
  install: ResolvedBundledPlugin,
  deps: BundledPluginProvisionerDeps,
  bundleManifestExists: (localPath: string) => boolean,
  verifiedManifest?: PaperclipPluginManifestV1,
): Promise<"upgrade_pending" | undefined> {
  try {
    // Managed reinstalls take the install path instead; this branch means the
    // operator's uninstall must be retained, including its status.
    if (install.distribution && existing.status === "uninstalled") return;
    if (!verifiedManifest && !bundleManifestExists(install.localPath)) return;
    const bundleManifest = verifiedManifest ?? await deps.loader.loadManifest(install.localPath);
    if (!bundleManifest) return;
    let requiresApproval = false;
    if (install.distribution) {
      assertDistributionManifestCapabilities(bundleManifest);
      const approved = new Set(existing.manifestJson.capabilities ?? []);
      requiresApproval = bundleManifest.capabilities.some((capability) => !approved.has(capability));
    }
    const rebindPackage = install.distribution && existing.packagePath !== install.localPath && existing.status !== "uninstalled";
    const refreshDistribution = install.distribution && !isDeepStrictEqual(bundleManifest, existing.manifestJson);
    if (bundleManifest.version === existing.version && !rebindPackage && !requiresApproval && !refreshDistribution) return;
    await deps.registry.update(existing.id, {
      version: bundleManifest.version,
      manifest: bundleManifest,
      ...(rebindPackage ? { packagePath: install.localPath } : {}),
      // Persist the replacement and its approval gate in one write. A crash
      // between separate manifest/status updates must never grant capabilities.
      ...(requiresApproval ? { status: "upgrade_pending" as const } : {}),
    });
    deps.logger.info(
      {
        pluginKey: install.pluginKey,
        fromVersion: existing.version,
        toVersion: bundleManifest.version,
        ...(rebindPackage ? { packagePath: install.localPath } : {}),
      },
      "reconciled bundled plugin manifest to the shipped bundle version",
    );
    if (requiresApproval) return "upgrade_pending";
  } catch (err) {
    deps.logger.error(
      { err, pluginKey: install.pluginKey },
      "Failed to reconcile bundled plugin manifest; continuing boot with the stored manifest",
    );
  }
}

/**
 * Re-enable a bundled plugin that a previous boot left in `error`.
 *
 * `error` is not an operator choice: the loader records it when activation
 * fails (for example the worker's `initialize` RPC timed out once) and it
 * also switches off the worker's auto-restart. Every automatic path
 * afterwards (`loadAll()`, the lazy worker recovery, the run lease) only
 * considers `ready` plugins, so a bundled plugin in `error` stays unusable
 * across restarts until an operator enables it by hand, and every run that
 * needs its provider fails with "that plugin is currently error". The bundle
 * ships with the release image and is expected to work, so one fresh attempt
 * per boot is the right default: the row goes back to `ready` (with its
 * `lastError` cleared), and the startup `loadAll()` activates it. If
 * activation fails again the loader marks `error` again and nothing retries
 * until the next boot, so this cannot loop within one process.
 *
 * This is a plain registry status reset, not `lifecycle.enable()`: the
 * lifecycle call would emit `plugin.enabled` before `loadAll()` has started
 * the worker, and a consumer of that event (the dev watcher, activity
 * listeners) would act on a plugin that may still fail to activate.
 * Activation, and its own events, stay with `loadAll()`.
 *
 * Fail-safe like the rest of the provisioner: a failed status reset is
 * logged and boot continues with the plugin unavailable.
 */
async function reenableErroredBundledPlugin(
  existing: RegistryPluginRow,
  install: ResolvedBundledPlugin,
  deps: BundledPluginProvisionerDeps,
): Promise<void> {
  deps.logger.warn(
    {
      pluginId: existing.id,
      pluginKey: install.pluginKey,
      lastError: existing.lastError ?? null,
    },
    "bundled plugin is in error status from a previous activation; re-enabling it for this boot",
  );
  try {
    await deps.registry.updateStatus(existing.id, { status: "ready", lastError: null });
    deps.logger.info(
      { pluginId: existing.id, pluginKey: install.pluginKey },
      "bundled plugin reset to ready; the startup loader will activate it",
    );
  } catch (err) {
    deps.logger.error(
      { err, pluginId: existing.id, pluginKey: install.pluginKey },
      "Failed to re-enable errored bundled plugin; continuing boot (degraded: plugin unavailable)",
    );
  }
}

/**
 * Ensure each resolved bundled plugin is installed and loaded.
 *
 * Same mechanism the kubernetes bundle has always used: in-process
 * `loader.installPlugin({ localPath })` at boot — no HTTP, no user, no
 * role. Fully fail-safe per entry: any disk/install/load
 * failure is caught, logged, and swallowed so boot always completes.
 *
 * Skip semantics:
 * - A plugin present in any non-uninstalled state is skipped, so an
 *   operator-disabled plugin is not silently re-enabled on reboot. Before the
 *   skip, the persisted manifest is reconciled to the shipped bundle version
 *   (see `reconcileBundledPluginManifest`).
 * - The one exception is `error`, which the loader sets when an activation
 *   fails and which no automatic path ever clears. A bundled plugin in
 *   `error` is moved back to `ready` once per boot so `loadAll()` gets a
 *   fresh attempt (see `reenableErroredBundledPlugin`).
 * - A soft-uninstalled plugin is reinstalled only when
 *   `reinstallUninstalled` is set (managed mode, where the control plane
 *   owns provisioning). Self-hosted keeps the pre-refactor behavior of
 *   leaving an operator's uninstall alone.
 */
export async function ensureBundledPlugins(
  installs: readonly ResolvedBundledPlugin[],
  deps: BundledPluginProvisionerDeps,
  opts: { reinstallUninstalled: boolean },
): Promise<void> {
  const bundleManifestExists = deps.bundleManifestExists ?? defaultBundleManifestExists;
  for (const install of installs) {
    try {
      let verifiedManifest: PaperclipPluginManifestV1 | undefined;
      if (install.distribution) {
        const manifest = await deps.loader.loadManifest(install.localPath);
        if (manifest?.id !== install.pluginKey || manifest.version !== install.distribution.version) {
          throw new Error("Distribution manifest does not match its catalog identity/version");
        }
        verifiedManifest = manifest;
      }
      const existing = await deps.registry.getByKey(install.pluginKey);
      if (existing && (existing.status !== "uninstalled" || !opts.reinstallUninstalled)) {
        // The bundle ships with the release image, so its manifest is the
        // source of truth for a present plugin. Reconcile the persisted
        // manifest when the shipped bundle declares a different version. Without
        // this step a manifest capability added to a bundle never reaches an
        // existing install, because the auto-install below skips a present
        // plugin. Distribution entries also replace a legacy/npm package path
        // before loadAll resolves the worker. Configuration and status stay put.
        const reconciledStatus = await reconcileBundledPluginManifest(existing, install, deps, bundleManifestExists, verifiedManifest);
        if (reconciledStatus === "upgrade_pending") continue;
        if (existing.status === "error") {
          await reenableErroredBundledPlugin(existing, install, deps);
          continue;
        }
        deps.logger.info(
          { pluginKey: install.pluginKey, status: existing.status },
          "bundled plugin already present; skipping auto-install",
        );
        continue;
      }
      // Skip silently when the bundle is absent (e.g. local dev or an image
      // built without the plugin). Not an error condition.
      if (!verifiedManifest && !bundleManifestExists(install.localPath)) {
        deps.logger.info(
          { pluginKey: install.pluginKey, pluginPath: install.localPath },
          "bundled plugin bundle not present; skipping auto-install",
        );
        continue;
      }
      deps.logger.info(
        { pluginKey: install.pluginKey, pluginPath: install.localPath },
        "auto-installing bundled plugin",
      );
      const discovered = await deps.loader.installPlugin({ localPath: install.localPath });
      if (!discovered.manifest) {
        deps.logger.error(
          { pluginKey: install.pluginKey },
          "bundled plugin installed but manifest is missing",
        );
        continue;
      }
      // Transition installed -> ready. Whether this also starts the worker
      // depends on the injected lifecycle manager: one built with a
      // runtime-capable loader activates here; the boot-time manager in
      // app.ts is not, so at startup this only records `ready` and the
      // worker is started exactly once by the subsequent loader.loadAll().
      const installed = await deps.registry.getByKey(discovered.manifest.id);
      if (installed) {
        await deps.lifecycle.load(installed.id);
        deps.logger.info(
          { pluginId: installed.id, pluginKey: installed.pluginKey },
          "bundled plugin auto-installed and loaded",
        );
      } else {
        deps.logger.error(
          { pluginKey: install.pluginKey },
          "bundled plugin installed but not found in registry",
        );
      }
    } catch (err) {
      deps.logger.error(
        { err, pluginKey: install.pluginKey },
        "Failed to auto-install bundled plugin; continuing boot (degraded: plugin unavailable)",
      );
    }
  }
}