PaperClipAI/server/src/worktree-config.ts

import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { isDeepStrictEqual } from "node:util";
import {
  mergePaperclipConfig,
  paperclipConfigSchema,
  type PaperclipConfig,
} from "@paperclipai/shared";
import { updateEnvFileContents, writeEnvFileAtomicallyIfChanged } from "@paperclipai/shared/env-file";
import { resolvePaperclipConfigPath, resolvePaperclipEnvPath } from "./paths.js";

function nonEmpty(value: string | null | undefined): string | null {
  return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}

function expandHomePrefix(value: string): string {
  if (value === "~") return os.homedir();
  if (value.startsWith("~/")) return path.resolve(os.homedir(), value.slice(2));
  return value;
}

function resolveHomeAwarePath(value: string): string {
  return path.resolve(expandHomePrefix(value));
}

function sanitizeWorktreeInstanceId(rawValue: string): string {
  const trimmed = rawValue.trim().toLowerCase();
  const normalized = trimmed
    .replace(/[^a-z0-9_-]+/g, "-")
    .replace(/-+/g, "-")
    .replace(/^[-_]+|[-_]+$/g, "");
  return normalized || "worktree";
}

function rewriteLocalUrlPort(rawUrl: string | undefined, port: number): string | undefined {
  if (!rawUrl) return undefined;
  try {
    const parsed = new URL(rawUrl);
    // The URL API normalizes default ports like :80/:443 to "", so treat them as stable URLs.
    if (!parsed.port) return rawUrl;
    parsed.port = String(port);
    return parsed.toString();
  } catch {
    return rawUrl;
  }
}

function parseEnvFile(contents: string): Record<string, string> {
  const entries: Record<string, string> = {};

  for (const rawLine of contents.split(/\r?\n/)) {
    const line = rawLine.trim();
    if (!line || line.startsWith("#")) continue;

    const match = rawLine.match(/^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$/);
    if (!match) continue;

    const [, key, rawValue] = match;
    const value = rawValue.trim();
    if (!value) {
      entries[key] = "";
      continue;
    }

    if (
      (value.startsWith("\"") && value.endsWith("\"")) ||
      (value.startsWith("'") && value.endsWith("'"))
    ) {
      entries[key] = value.slice(1, -1);
      continue;
    }

    entries[key] = value.replace(/\s+#.*$/, "").trim();
  }

  return entries;
}

function readEnvEntries(envPath: string): Record<string, string> {
  if (!fs.existsSync(envPath)) return {};
  return parseEnvFile(fs.readFileSync(envPath, "utf8"));
}

function emptyWorktreeEnvFileContents(): string {
  return [
    "# Paperclip environment variables",
    "# Generated by Paperclip worktree repair",
    "",
  ].join("\n");
}

function isPathInside(candidatePath: string, rootPath: string): boolean {
  const candidate = path.resolve(candidatePath);
  const root = path.resolve(rootPath);
  return candidate === root || candidate.startsWith(`${root}${path.sep}`);
}

type WorktreeRuntimeContext = {
  configPath: string;
  envPath: string;
  worktreeName: string;
  instanceId: string;
  homeDir: string;
  instanceRoot: string;
  contextPath: string;
  embeddedPostgresDataDir: string;
  backupDir: string;
  logDir: string;
  storageDir: string;
  secretsKeyFilePath: string;
};

type WorktreePortRegistry = {
  version: 1;
  configPaths: string[];
};

const WORKTREE_PORT_REGISTRY_FILE = "worktree-port-reservations.json";
const WORKTREE_PORT_REGISTRY_LOCK_DIR = ".worktree-port-reservations.lock";
const WORKTREE_PORT_REGISTRY_LOCK_STALE_MS = 5_000;
const WORKTREE_PORT_REGISTRY_LOCK_TIMEOUT_MS = 10_000;
const sleepSyncBuffer = new Int32Array(new SharedArrayBuffer(4));

function sleepSync(durationMs: number): void {
  Atomics.wait(sleepSyncBuffer, 0, 0, durationMs);
}

function withWorktreePortRegistryLock<T>(homeDir: string, run: () => T): T {
  fs.mkdirSync(homeDir, { recursive: true });
  const lockPath = path.resolve(homeDir, WORKTREE_PORT_REGISTRY_LOCK_DIR);
  const deadline = Date.now() + WORKTREE_PORT_REGISTRY_LOCK_TIMEOUT_MS;

  while (true) {
    try {
      fs.mkdirSync(lockPath);
      break;
    } catch (error) {
      const code = error instanceof Error && "code" in error ? error.code : null;
      if (code !== "EEXIST") throw error;

      try {
        const ageMs = Date.now() - fs.statSync(lockPath).mtimeMs;
        if (ageMs > WORKTREE_PORT_REGISTRY_LOCK_STALE_MS) {
          fs.rmSync(lockPath, { recursive: true, force: true });
          continue;
        }
      } catch {
        continue;
      }

      if (Date.now() >= deadline) {
        throw new Error(`Timed out waiting for worktree port reservation lock at ${lockPath}`);
      }
      sleepSync(25);
    }
  }

  try {
    return run();
  } finally {
    fs.rmSync(lockPath, { recursive: true, force: true });
  }
}

function readWorktreePortRegistry(homeDir: string): Set<string> {
  const registryPath = path.resolve(homeDir, WORKTREE_PORT_REGISTRY_FILE);
  if (!fs.existsSync(registryPath)) return new Set();

  try {
    const parsed = JSON.parse(fs.readFileSync(registryPath, "utf8")) as Partial<WorktreePortRegistry>;
    if (parsed.version !== 1 || !Array.isArray(parsed.configPaths)) return new Set();
    return new Set(
      parsed.configPaths
        .filter((configPath): configPath is string => typeof configPath === "string" && configPath.length > 0)
        .map((configPath) => path.resolve(configPath)),
    );
  } catch {
    return new Set();
  }
}

function writeWorktreePortRegistry(homeDir: string, configPaths: Iterable<string>): void {
  const registryPath = path.resolve(homeDir, WORKTREE_PORT_REGISTRY_FILE);
  const persistedPaths = Array.from(new Set(Array.from(configPaths, (configPath) => path.resolve(configPath))))
    .filter((configPath) => fs.existsSync(configPath))
    .sort();
  const registry: WorktreePortRegistry = {
    version: 1,
    configPaths: persistedPaths,
  };
  const temporaryPath = `${registryPath}.${process.pid}.tmp`;
  fs.writeFileSync(temporaryPath, `${JSON.stringify(registry, null, 2)}\n`, { mode: 0o600 });
  fs.renameSync(temporaryPath, registryPath);
}

function resolveWorktreeRuntimeContext(
  env: NodeJS.ProcessEnv,
  overrideConfigPath?: string,
): WorktreeRuntimeContext | null {
  if (env.PAPERCLIP_IN_WORKTREE !== "true") return null;

  const configPath = resolvePaperclipConfigPath(overrideConfigPath);
  const envPath = resolvePaperclipEnvPath(configPath);
  const persistedEnv = readEnvEntries(envPath);

  // PAPERCLIP_IN_WORKTREE can leak in from a parent process or a sourced env
  // file while config resolution still points at a non-worktree target (for
  // example the default instance under <home>/instances/default). Only adopt
  // a target as a worktree when its config sits in a `<root>/.paperclip/`
  // layout and its own persisted env already declares it a worktree;
  // otherwise the repair would rewrite main-instance config and env files.
  if (path.basename(path.dirname(configPath)) !== ".paperclip") return null;
  if (persistedEnv.PAPERCLIP_IN_WORKTREE !== "true") return null;

  const persistedConfigPath = nonEmpty(persistedEnv.PAPERCLIP_CONFIG);
  const persistedConfigLooksStale =
    persistedConfigPath !== null &&
    path.resolve(expandHomePrefix(persistedConfigPath)) !== path.resolve(configPath) &&
    !fs.existsSync(resolveHomeAwarePath(persistedConfigPath));
  const stablePersistedEnv = persistedConfigLooksStale ? {} : persistedEnv;
  const worktreeRoot = path.resolve(path.dirname(configPath), "..");
  const worktreeName =
    nonEmpty(stablePersistedEnv.PAPERCLIP_WORKTREE_NAME) ??
    nonEmpty(env.PAPERCLIP_WORKTREE_NAME) ??
    path.basename(worktreeRoot);
  const instanceId =
    nonEmpty(stablePersistedEnv.PAPERCLIP_INSTANCE_ID) ??
    nonEmpty(env.PAPERCLIP_INSTANCE_ID) ??
    sanitizeWorktreeInstanceId(worktreeName);
  const homeDir = resolveHomeAwarePath(
    nonEmpty(stablePersistedEnv.PAPERCLIP_HOME) ??
      nonEmpty(env.PAPERCLIP_HOME) ??
      nonEmpty(env.PAPERCLIP_WORKTREES_DIR) ??
      "~/.paperclip-worktrees",
  );
  const instanceRoot = path.resolve(homeDir, "instances", instanceId);

  return {
    configPath,
    envPath,
    worktreeName,
    instanceId,
    homeDir,
    instanceRoot,
    contextPath: path.resolve(homeDir, "context.json"),
    embeddedPostgresDataDir: path.resolve(instanceRoot, "db"),
    backupDir: path.resolve(instanceRoot, "data", "backups"),
    logDir: path.resolve(instanceRoot, "logs"),
    storageDir: path.resolve(instanceRoot, "data", "storage"),
    secretsKeyFilePath: path.resolve(instanceRoot, "secrets", "master.key"),
  };
}

function atomicWriteFile(filePath: string, contents: string): void {
  let attempt = 0;

  while (true) {
    const temporaryPath = `${filePath}.tmp-${process.pid}-${attempt}`;
    attempt += 1;
    let fileDescriptor: number | null = null;
    try {
      fileDescriptor = fs.openSync(temporaryPath, "wx", 0o600);
      fs.writeFileSync(fileDescriptor, contents, "utf8");
      fs.fsyncSync(fileDescriptor);
      fs.closeSync(fileDescriptor);
      fileDescriptor = null;
      fs.renameSync(temporaryPath, filePath);
      let directoryDescriptor: number | null = null;
      try {
        directoryDescriptor = fs.openSync(path.dirname(filePath), "r");
        fs.fsyncSync(directoryDescriptor);
      } catch (error) {
        const code = error instanceof Error && "code" in error ? error.code : null;
        if (
          process.platform !== "win32" ||
          !["EACCES", "EINVAL", "EISDIR", "ENOTSUP", "EPERM"].includes(String(code))
        ) {
          throw error;
        }
      } finally {
        if (directoryDescriptor !== null) fs.closeSync(directoryDescriptor);
      }
      return;
    } catch (error) {
      if (fileDescriptor !== null) fs.closeSync(fileDescriptor);
      fs.rmSync(temporaryPath, { force: true });
      const code = error instanceof Error && "code" in error ? error.code : null;
      if (code === "EEXIST") continue;
      throw error;
    }
  }
}

function writeConfigFile(configPath: string, config: PaperclipConfig): boolean {
  fs.mkdirSync(path.dirname(configPath), { recursive: true });
  const update = paperclipConfigSchema.parse(config);
  const source = fs.existsSync(configPath)
    ? paperclipConfigSchema.parse(JSON.parse(fs.readFileSync(configPath, "utf8")))
    : null;
  const nextConfig = source
    ? paperclipConfigSchema.parse(mergePaperclipConfig(source, update))
    : update;

  if (source && isDeepStrictEqual(source, nextConfig)) return false;

  atomicWriteFile(configPath, JSON.stringify(nextConfig, null, 2) + "\n");
  return true;
}

function resolveRepoManagedWorktreesRoot(worktreeRoot: string): string | null {
  const normalized = path.resolve(worktreeRoot);
  const marker = `${path.sep}.paperclip${path.sep}worktrees${path.sep}`;
  const index = normalized.indexOf(marker);
  if (index === -1) return null;
  const repoRoot = normalized.slice(0, index);
  return path.resolve(repoRoot, ".paperclip", "worktrees");
}

function collectSiblingWorktreePorts(
  context: WorktreeRuntimeContext,
  registeredConfigPaths: Iterable<string> = [],
): {
  serverPorts: Set<number>;
  databasePorts: Set<number>;
  configPaths: Set<string>;
} {
  const serverPorts = new Set<number>();
  const databasePorts = new Set<number>();
  const siblingConfigPaths = new Set<string>();
  for (const configPath of registeredConfigPaths) {
    const resolvedConfigPath = path.resolve(configPath);
    if (resolvedConfigPath !== path.resolve(context.configPath) && fs.existsSync(resolvedConfigPath)) {
      siblingConfigPaths.add(resolvedConfigPath);
    }
  }
  const instancesDir = path.resolve(context.homeDir, "instances");
  if (fs.existsSync(instancesDir)) {
    for (const entry of fs.readdirSync(instancesDir, { withFileTypes: true })) {
      if (!entry.isDirectory() || entry.name === context.instanceId) continue;

      const siblingConfigPath = path.resolve(instancesDir, entry.name, "config.json");
      if (fs.existsSync(siblingConfigPath)) {
        siblingConfigPaths.add(siblingConfigPath);
      }
    }
  }

  const repoManagedWorktreesRoot = resolveRepoManagedWorktreesRoot(path.dirname(context.configPath));
  if (repoManagedWorktreesRoot && fs.existsSync(repoManagedWorktreesRoot)) {
    for (const entry of fs.readdirSync(repoManagedWorktreesRoot, { withFileTypes: true })) {
      if (!entry.isDirectory()) continue;

      const siblingConfigPath = path.resolve(repoManagedWorktreesRoot, entry.name, ".paperclip", "config.json");
      if (path.resolve(siblingConfigPath) === path.resolve(context.configPath)) continue;
      if (fs.existsSync(siblingConfigPath)) {
        siblingConfigPaths.add(siblingConfigPath);
      }
    }
  }

  for (const siblingConfigPath of siblingConfigPaths) {
    try {
      const siblingConfig = JSON.parse(fs.readFileSync(siblingConfigPath, "utf8")) as PaperclipConfig;
      if (Number.isInteger(siblingConfig.server.port) && siblingConfig.server.port > 0) {
        serverPorts.add(siblingConfig.server.port);
      }
      if (
        siblingConfig.database.mode === "embedded-postgres" &&
        Number.isInteger(siblingConfig.database.embeddedPostgresPort) &&
        siblingConfig.database.embeddedPostgresPort > 0
      ) {
        databasePorts.add(siblingConfig.database.embeddedPostgresPort);
      }
    } catch {
      // Ignore sibling configs that are missing or malformed.
    }
  }

  return { serverPorts, databasePorts, configPaths: siblingConfigPaths };
}

function findNextUnclaimedPort(preferredPort: number, claimedPorts: Set<number>): number {
  let port = Math.max(1, Math.trunc(preferredPort));
  while (claimedPorts.has(port)) {
    port += 1;
  }
  return port;
}

function buildIsolatedWorktreeConfig(
  config: PaperclipConfig,
  context: WorktreeRuntimeContext,
  portOverrides?: {
    serverPort?: number;
    databasePort?: number;
  },
): PaperclipConfig {
  const serverPort = portOverrides?.serverPort ?? config.server.port;
  const databasePort =
    config.database.mode === "embedded-postgres"
      ? portOverrides?.databasePort ?? config.database.embeddedPostgresPort
      : undefined;
  const nextConfig: PaperclipConfig = {
    ...config,
    database: {
      ...config.database,
      ...(config.database.mode === "embedded-postgres"
        ? {
            embeddedPostgresDataDir: context.embeddedPostgresDataDir,
            embeddedPostgresPort: databasePort ?? config.database.embeddedPostgresPort,
            backup: {
              ...config.database.backup,
              enabled: false,
              dir: context.backupDir,
            },
          }
        : {}),
    },
    server: {
      ...config.server,
      port: serverPort,
    },
    logging: {
      ...config.logging,
      logDir: context.logDir,
    },
    storage: {
      ...config.storage,
      localDisk: {
        ...config.storage.localDisk,
        baseDir: context.storageDir,
      },
    },
    secrets: {
      ...config.secrets,
      localEncrypted: {
        ...config.secrets.localEncrypted,
        keyFilePath: context.secretsKeyFilePath,
      },
    },
  };

  if (config.auth.baseUrlMode === "explicit" && config.auth.publicBaseUrl) {
    nextConfig.auth = {
      ...config.auth,
      publicBaseUrl: rewriteLocalUrlPort(config.auth.publicBaseUrl, serverPort),
    };
  }

  return nextConfig;
}

function needsWorktreeConfigRepair(
  config: PaperclipConfig,
  context: WorktreeRuntimeContext,
): boolean {
  if (config.database.mode === "embedded-postgres") {
    if (config.database.backup.enabled) {
      return true;
    }
    if (!isPathInside(config.database.embeddedPostgresDataDir, context.instanceRoot)) {
      return true;
    }
    if (!isPathInside(config.database.backup.dir, context.instanceRoot)) {
      return true;
    }
  }

  if (!isPathInside(config.logging.logDir, context.instanceRoot)) {
    return true;
  }
  if (!isPathInside(config.storage.localDisk.baseDir, context.instanceRoot)) {
    return true;
  }
  if (!isPathInside(config.secrets.localEncrypted.keyFilePath, context.instanceRoot)) {
    return true;
  }

  return false;
}

export function applyRuntimePortSelectionToConfig(
  config: PaperclipConfig,
  input: {
    serverPort: number;
    databasePort?: number | null;
    allowServerPortWrite?: boolean;
    allowDatabasePortWrite?: boolean;
  },
): { config: PaperclipConfig; changed: boolean } {
  let changed = false;
  let nextConfig = config;

  if (input.allowServerPortWrite !== false && config.server.port !== input.serverPort) {
    nextConfig = {
      ...nextConfig,
      server: {
        ...nextConfig.server,
        port: input.serverPort,
      },
    };
    changed = true;
  }

  if (
    input.allowDatabasePortWrite !== false &&
    nextConfig.database.mode === "embedded-postgres" &&
    typeof input.databasePort === "number" &&
    nextConfig.database.embeddedPostgresPort !== input.databasePort
  ) {
    nextConfig = {
      ...nextConfig,
      database: {
        ...nextConfig.database,
        embeddedPostgresPort: input.databasePort,
      },
    };
    changed = true;
  }

  if (nextConfig.auth.baseUrlMode === "explicit" && nextConfig.auth.publicBaseUrl) {
    const rewritten = rewriteLocalUrlPort(nextConfig.auth.publicBaseUrl, input.serverPort);
    if (rewritten && rewritten !== nextConfig.auth.publicBaseUrl) {
      nextConfig = {
        ...nextConfig,
        auth: {
          ...nextConfig.auth,
          publicBaseUrl: rewritten,
        },
      };
      changed = true;
    }
  }

  return { config: nextConfig, changed };
}

export function maybeRepairLegacyWorktreeConfigAndEnvFiles(): {
  repairedConfig: boolean;
  repairedEnv: boolean;
} {
  const context = resolveWorktreeRuntimeContext(process.env);
  if (!context) {
    return { repairedConfig: false, repairedEnv: false };
  }

  process.env.PAPERCLIP_HOME = context.homeDir;
  process.env.PAPERCLIP_INSTANCE_ID = context.instanceId;
  process.env.PAPERCLIP_CONFIG = context.configPath;
  process.env.PAPERCLIP_CONTEXT = context.contextPath;
  process.env.PAPERCLIP_WORKTREE_NAME = context.worktreeName;

  let repairedConfig = false;
  if (fs.existsSync(context.configPath)) {
    try {
      const runtimeConfig = withWorktreePortRegistryLock(context.homeDir, () => {
        const parsed = JSON.parse(fs.readFileSync(context.configPath, "utf8")) as PaperclipConfig;
        let selectedConfig = parsed;
        const registeredConfigPaths = readWorktreePortRegistry(context.homeDir);
        const siblingPorts = collectSiblingWorktreePorts(context, registeredConfigPaths);
        const serverPortCollision = siblingPorts.serverPorts.has(parsed.server.port);
        const databasePortCollision =
          parsed.database.mode === "embedded-postgres" &&
          siblingPorts.databasePorts.has(parsed.database.embeddedPostgresPort);

        if (needsWorktreeConfigRepair(parsed, context) || serverPortCollision || databasePortCollision) {
          const selectedServerPort = findNextUnclaimedPort(
            parsed.server.port === 3100 ? 3101 : parsed.server.port,
            siblingPorts.serverPorts,
          );
          const selectedDatabasePort =
            parsed.database.mode === "embedded-postgres"
              ? findNextUnclaimedPort(
                  parsed.database.embeddedPostgresPort === 54329
                    ? 54330
                    : parsed.database.embeddedPostgresPort,
                  new Set([...siblingPorts.databasePorts, selectedServerPort]),
                )
              : undefined;

          selectedConfig = buildIsolatedWorktreeConfig(parsed, context, {
            serverPort: selectedServerPort,
            databasePort: selectedDatabasePort,
          });
          writeConfigFile(context.configPath, selectedConfig);
          repairedConfig = true;

          if (serverPortCollision || databasePortCollision) {
            console.warn(
              [
                `Worktree port conflict detected for ${context.worktreeName}; updated and persisted workspace ports.`,
                ...(serverPortCollision
                  ? [`server: ${parsed.server.port} -> ${selectedServerPort}`]
                  : []),
                ...(databasePortCollision && parsed.database.mode === "embedded-postgres"
                  ? [`database: ${parsed.database.embeddedPostgresPort} -> ${selectedDatabasePort}`]
                  : []),
              ].join(" "),
            );
          }
        }

        writeWorktreePortRegistry(context.homeDir, [
          ...registeredConfigPaths,
          ...siblingPorts.configPaths,
          context.configPath,
        ]);
        return selectedConfig;
      });

      if (
        !nonEmpty(process.env.PORT)
        && Number.isInteger(runtimeConfig.server.port)
        && runtimeConfig.server.port > 0
      ) {
        process.env.PORT = String(runtimeConfig.server.port);
      }
    } catch {
      // Leave invalid configs to the normal startup validation path.
    }
  }

  const managedEnvEntries: Record<string, string> = {
    PAPERCLIP_HOME: context.homeDir,
    PAPERCLIP_INSTANCE_ID: context.instanceId,
    PAPERCLIP_CONFIG: context.configPath,
    PAPERCLIP_CONTEXT: context.contextPath,
    PAPERCLIP_IN_WORKTREE: "true",
    PAPERCLIP_DB_BACKUP_ENABLED: "false",
    PAPERCLIP_WORKTREE_NAME: context.worktreeName,
  };

  process.env.PAPERCLIP_DB_BACKUP_ENABLED = "false";

  const existingContents = fs.existsSync(context.envPath)
    ? fs.readFileSync(context.envPath, "utf8")
    : null;
  const repairedContents = updateEnvFileContents(
    existingContents ?? emptyWorktreeEnvFileContents(),
    managedEnvEntries,
    { valueEncoding: "json" },
  );
  const repairedEnv = existingContents !== repairedContents;

  if (repairedEnv) {
    writeEnvFileAtomicallyIfChanged(context.envPath, existingContents, repairedContents);
  }

  return { repairedConfig, repairedEnv };
}

function isPortPinnedByRuntimeEnv(rawValue: string | null | undefined, selectedPort: number): boolean {
  const normalized = nonEmpty(rawValue);
  if (!normalized) return false;
  const parsedPort = Number(normalized);
  if (!Number.isInteger(parsedPort) || parsedPort <= 0) return true;
  return parsedPort === selectedPort;
}

export function maybePersistWorktreeRuntimePorts(input: {
  serverPort: number;
  databasePort?: number | null;
}): void {
  const context = resolveWorktreeRuntimeContext(process.env);
  if (!context || !fs.existsSync(context.configPath)) return;

  let fileConfig: PaperclipConfig;
  try {
    fileConfig = JSON.parse(fs.readFileSync(context.configPath, "utf8")) as PaperclipConfig;
  } catch {
    return;
  }

  const { config, changed } = applyRuntimePortSelectionToConfig(fileConfig, {
    serverPort: input.serverPort,
    databasePort: input.databasePort,
    allowServerPortWrite: !isPortPinnedByRuntimeEnv(process.env.PORT, input.serverPort),
    allowDatabasePortWrite: !nonEmpty(process.env.DATABASE_URL),
  });

  if (changed) {
    writeConfigFile(context.configPath, config);
  }
}