Files
PaperClipAI/server/src/worktree-config.ts
T

711 lines
23 KiB
TypeScript

import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { randomUUID } from "node:crypto";
import { isDeepStrictEqual } from "node:util";
import {
paperclipConfigSchema,
type PaperclipConfig,
} from "@paperclipai/shared";
import { mergeConfigValues } from "@paperclipai/shared/config-persistence";
import { resolvePaperclipConfigPath, resolvePaperclipEnvPath } from "./paths.js";
function nonEmpty(value: string | null | undefined): string | null {
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}
function expandHomePrefix(value: string): string {
if (value === "~") return os.homedir();
if (value.startsWith("~/")) return path.resolve(os.homedir(), value.slice(2));
return value;
}
function resolveHomeAwarePath(value: string): string {
return path.resolve(expandHomePrefix(value));
}
function sanitizeWorktreeInstanceId(rawValue: string): string {
const trimmed = rawValue.trim().toLowerCase();
const normalized = trimmed
.replace(/[^a-z0-9_-]+/g, "-")
.replace(/-+/g, "-")
.replace(/^[-_]+|[-_]+$/g, "");
return normalized || "worktree";
}
function rewriteLocalUrlPort(rawUrl: string | undefined, port: number): string | undefined {
if (!rawUrl) return undefined;
try {
const parsed = new URL(rawUrl);
// The URL API normalizes default ports like :80/:443 to "", so treat them as stable URLs.
if (!parsed.port) return rawUrl;
parsed.port = String(port);
return parsed.toString();
} catch {
return rawUrl;
}
}
function parseEnvFile(contents: string): Record<string, string> {
const entries: Record<string, string> = {};
for (const rawLine of contents.split(/\r?\n/)) {
const line = rawLine.trim();
if (!line || line.startsWith("#")) continue;
const match = rawLine.match(/^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$/);
if (!match) continue;
const [, key, rawValue] = match;
const value = rawValue.trim();
if (!value) {
entries[key] = "";
continue;
}
if (
(value.startsWith("\"") && value.endsWith("\"")) ||
(value.startsWith("'") && value.endsWith("'"))
) {
entries[key] = value.slice(1, -1);
continue;
}
entries[key] = value.replace(/\s+#.*$/, "").trim();
}
return entries;
}
function readEnvEntries(envPath: string): Record<string, string> {
if (!fs.existsSync(envPath)) return {};
return parseEnvFile(fs.readFileSync(envPath, "utf8"));
}
function formatEnvEntries(entries: Record<string, string>): string {
return [
"# Paperclip environment variables",
"# Generated by Paperclip worktree repair",
...Object.entries(entries).map(([key, value]) => `${key}=${JSON.stringify(value)}`),
"",
].join("\n");
}
function trailingEnvComment(rawValue: string): string {
let quote: "\"" | "'" | null = null;
let escaped = false;
for (let index = 0; index < rawValue.length; index += 1) {
const character = rawValue[index];
if (escaped) {
escaped = false;
continue;
}
if (quote === "\"" && character === "\\") {
escaped = true;
continue;
}
if (quote !== null) {
if (character === quote) quote = null;
continue;
}
if (character === "\"" || character === "'") {
quote = character;
continue;
}
if (character !== "#" || index === 0 || !/\s/.test(rawValue[index - 1] ?? "")) continue;
let commentStart = index;
while (commentStart > 0 && /\s/.test(rawValue[commentStart - 1] ?? "")) {
commentStart -= 1;
}
return rawValue.slice(commentStart);
}
return "";
}
function updateEnvFileContents(contents: string, entries: Record<string, string>): string {
const newline = contents.includes("\r\n") ? "\r\n" : "\n";
const missingEntries = new Map(Object.entries(entries));
const lines = contents.split(/\r?\n/).map((rawLine) => {
const match = rawLine.match(/^(\s*(?:export\s+)?)([A-Za-z_][A-Za-z0-9_]*)(\s*=\s*)(.*)$/);
if (!match) return rawLine;
const [, prefix, key, separator, rawValue] = match;
const value = entries[key];
if (value === undefined) return rawLine;
missingEntries.delete(key);
return `${prefix}${key}${separator}${JSON.stringify(value)}${trailingEnvComment(rawValue)}`;
});
const insertionIndex = lines.at(-1) === "" ? lines.length - 1 : lines.length;
lines.splice(
insertionIndex,
0,
...Array.from(missingEntries, ([key, value]) => `${key}=${JSON.stringify(value)}`),
);
return lines.join(newline);
}
function isPathInside(candidatePath: string, rootPath: string): boolean {
const candidate = path.resolve(candidatePath);
const root = path.resolve(rootPath);
return candidate === root || candidate.startsWith(`${root}${path.sep}`);
}
type WorktreeRuntimeContext = {
configPath: string;
envPath: string;
worktreeName: string;
instanceId: string;
homeDir: string;
instanceRoot: string;
contextPath: string;
embeddedPostgresDataDir: string;
backupDir: string;
logDir: string;
storageDir: string;
secretsKeyFilePath: string;
};
type WorktreePortRegistry = {
version: 1;
configPaths: string[];
};
const WORKTREE_PORT_REGISTRY_FILE = "worktree-port-reservations.json";
const WORKTREE_PORT_REGISTRY_LOCK_DIR = ".worktree-port-reservations.lock";
const WORKTREE_PORT_REGISTRY_LOCK_STALE_MS = 5_000;
const WORKTREE_PORT_REGISTRY_LOCK_TIMEOUT_MS = 10_000;
const sleepSyncBuffer = new Int32Array(new SharedArrayBuffer(4));
function sleepSync(durationMs: number): void {
Atomics.wait(sleepSyncBuffer, 0, 0, durationMs);
}
function withWorktreePortRegistryLock<T>(homeDir: string, run: () => T): T {
fs.mkdirSync(homeDir, { recursive: true });
const lockPath = path.resolve(homeDir, WORKTREE_PORT_REGISTRY_LOCK_DIR);
const deadline = Date.now() + WORKTREE_PORT_REGISTRY_LOCK_TIMEOUT_MS;
while (true) {
try {
fs.mkdirSync(lockPath);
break;
} catch (error) {
const code = error instanceof Error && "code" in error ? error.code : null;
if (code !== "EEXIST") throw error;
try {
const ageMs = Date.now() - fs.statSync(lockPath).mtimeMs;
if (ageMs > WORKTREE_PORT_REGISTRY_LOCK_STALE_MS) {
fs.rmSync(lockPath, { recursive: true, force: true });
continue;
}
} catch {
continue;
}
if (Date.now() >= deadline) {
throw new Error(`Timed out waiting for worktree port reservation lock at ${lockPath}`);
}
sleepSync(25);
}
}
try {
return run();
} finally {
fs.rmSync(lockPath, { recursive: true, force: true });
}
}
function readWorktreePortRegistry(homeDir: string): Set<string> {
const registryPath = path.resolve(homeDir, WORKTREE_PORT_REGISTRY_FILE);
if (!fs.existsSync(registryPath)) return new Set();
try {
const parsed = JSON.parse(fs.readFileSync(registryPath, "utf8")) as Partial<WorktreePortRegistry>;
if (parsed.version !== 1 || !Array.isArray(parsed.configPaths)) return new Set();
return new Set(
parsed.configPaths
.filter((configPath): configPath is string => typeof configPath === "string" && configPath.length > 0)
.map((configPath) => path.resolve(configPath)),
);
} catch {
return new Set();
}
}
function writeWorktreePortRegistry(homeDir: string, configPaths: Iterable<string>): void {
const registryPath = path.resolve(homeDir, WORKTREE_PORT_REGISTRY_FILE);
const persistedPaths = Array.from(new Set(Array.from(configPaths, (configPath) => path.resolve(configPath))))
.filter((configPath) => fs.existsSync(configPath))
.sort();
const registry: WorktreePortRegistry = {
version: 1,
configPaths: persistedPaths,
};
const temporaryPath = `${registryPath}.${process.pid}.tmp`;
fs.writeFileSync(temporaryPath, `${JSON.stringify(registry, null, 2)}\n`, { mode: 0o600 });
fs.renameSync(temporaryPath, registryPath);
}
function resolveWorktreeRuntimeContext(
env: NodeJS.ProcessEnv,
overrideConfigPath?: string,
): WorktreeRuntimeContext | null {
if (env.PAPERCLIP_IN_WORKTREE !== "true") return null;
const configPath = resolvePaperclipConfigPath(overrideConfigPath);
const envPath = resolvePaperclipEnvPath(configPath);
const persistedEnv = readEnvEntries(envPath);
// PAPERCLIP_IN_WORKTREE can leak in from a parent process or a sourced env
// file while config resolution still points at a non-worktree target (for
// example the default instance under <home>/instances/default). Only adopt
// a target as a worktree when its config sits in a `<root>/.paperclip/`
// layout and its own persisted env already declares it a worktree;
// otherwise the repair would rewrite main-instance config and env files.
if (path.basename(path.dirname(configPath)) !== ".paperclip") return null;
if (persistedEnv.PAPERCLIP_IN_WORKTREE !== "true") return null;
const persistedConfigPath = nonEmpty(persistedEnv.PAPERCLIP_CONFIG);
const persistedConfigLooksStale =
persistedConfigPath !== null &&
path.resolve(expandHomePrefix(persistedConfigPath)) !== path.resolve(configPath) &&
!fs.existsSync(resolveHomeAwarePath(persistedConfigPath));
const stablePersistedEnv = persistedConfigLooksStale ? {} : persistedEnv;
const worktreeRoot = path.resolve(path.dirname(configPath), "..");
const worktreeName =
nonEmpty(stablePersistedEnv.PAPERCLIP_WORKTREE_NAME) ??
nonEmpty(env.PAPERCLIP_WORKTREE_NAME) ??
path.basename(worktreeRoot);
const instanceId =
nonEmpty(stablePersistedEnv.PAPERCLIP_INSTANCE_ID) ??
nonEmpty(env.PAPERCLIP_INSTANCE_ID) ??
sanitizeWorktreeInstanceId(worktreeName);
const homeDir = resolveHomeAwarePath(
nonEmpty(stablePersistedEnv.PAPERCLIP_HOME) ??
nonEmpty(env.PAPERCLIP_HOME) ??
nonEmpty(env.PAPERCLIP_WORKTREES_DIR) ??
"~/.paperclip-worktrees",
);
const instanceRoot = path.resolve(homeDir, "instances", instanceId);
return {
configPath,
envPath,
worktreeName,
instanceId,
homeDir,
instanceRoot,
contextPath: path.resolve(homeDir, "context.json"),
embeddedPostgresDataDir: path.resolve(instanceRoot, "db"),
backupDir: path.resolve(instanceRoot, "data", "backups"),
logDir: path.resolve(instanceRoot, "logs"),
storageDir: path.resolve(instanceRoot, "data", "storage"),
secretsKeyFilePath: path.resolve(instanceRoot, "secrets", "master.key"),
};
}
function writeConfigFile(configPath: string, config: PaperclipConfig): boolean {
fs.mkdirSync(path.dirname(configPath), { recursive: true });
let sourceConfig: PaperclipConfig | null = null;
if (fs.existsSync(configPath)) {
const raw = JSON.parse(fs.readFileSync(configPath, "utf8")) as unknown;
sourceConfig = paperclipConfigSchema.parse(raw);
}
const merged = sourceConfig ? mergeConfigValues(sourceConfig, config) : config;
const parsed = paperclipConfigSchema.parse(merged);
if (sourceConfig && isDeepStrictEqual(sourceConfig, parsed)) return false;
const temporaryPath = path.join(
path.dirname(configPath),
`.${path.basename(configPath)}.tmp-${process.pid}-${randomUUID()}`,
);
try {
fs.writeFileSync(temporaryPath, JSON.stringify(parsed, null, 2) + "\n", {
mode: 0o600,
flag: "wx",
});
fs.renameSync(temporaryPath, configPath);
} finally {
fs.rmSync(temporaryPath, { force: true });
}
return true;
}
function resolveRepoManagedWorktreesRoot(worktreeRoot: string): string | null {
const normalized = path.resolve(worktreeRoot);
const marker = `${path.sep}.paperclip${path.sep}worktrees${path.sep}`;
const index = normalized.indexOf(marker);
if (index === -1) return null;
const repoRoot = normalized.slice(0, index);
return path.resolve(repoRoot, ".paperclip", "worktrees");
}
function collectSiblingWorktreePorts(
context: WorktreeRuntimeContext,
registeredConfigPaths: Iterable<string> = [],
): {
serverPorts: Set<number>;
databasePorts: Set<number>;
configPaths: Set<string>;
} {
const serverPorts = new Set<number>();
const databasePorts = new Set<number>();
const siblingConfigPaths = new Set<string>();
for (const configPath of registeredConfigPaths) {
const resolvedConfigPath = path.resolve(configPath);
if (resolvedConfigPath !== path.resolve(context.configPath) && fs.existsSync(resolvedConfigPath)) {
siblingConfigPaths.add(resolvedConfigPath);
}
}
const instancesDir = path.resolve(context.homeDir, "instances");
if (fs.existsSync(instancesDir)) {
for (const entry of fs.readdirSync(instancesDir, { withFileTypes: true })) {
if (!entry.isDirectory() || entry.name === context.instanceId) continue;
const siblingConfigPath = path.resolve(instancesDir, entry.name, "config.json");
if (fs.existsSync(siblingConfigPath)) {
siblingConfigPaths.add(siblingConfigPath);
}
}
}
const repoManagedWorktreesRoot = resolveRepoManagedWorktreesRoot(path.dirname(context.configPath));
if (repoManagedWorktreesRoot && fs.existsSync(repoManagedWorktreesRoot)) {
for (const entry of fs.readdirSync(repoManagedWorktreesRoot, { withFileTypes: true })) {
if (!entry.isDirectory()) continue;
const siblingConfigPath = path.resolve(repoManagedWorktreesRoot, entry.name, ".paperclip", "config.json");
if (path.resolve(siblingConfigPath) === path.resolve(context.configPath)) continue;
if (fs.existsSync(siblingConfigPath)) {
siblingConfigPaths.add(siblingConfigPath);
}
}
}
for (const siblingConfigPath of siblingConfigPaths) {
try {
const siblingConfig = JSON.parse(fs.readFileSync(siblingConfigPath, "utf8")) as PaperclipConfig;
if (Number.isInteger(siblingConfig.server.port) && siblingConfig.server.port > 0) {
serverPorts.add(siblingConfig.server.port);
}
if (
siblingConfig.database.mode === "embedded-postgres" &&
Number.isInteger(siblingConfig.database.embeddedPostgresPort) &&
siblingConfig.database.embeddedPostgresPort > 0
) {
databasePorts.add(siblingConfig.database.embeddedPostgresPort);
}
} catch {
// Ignore sibling configs that are missing or malformed.
}
}
return { serverPorts, databasePorts, configPaths: siblingConfigPaths };
}
function findNextUnclaimedPort(preferredPort: number, claimedPorts: Set<number>): number {
let port = Math.max(1, Math.trunc(preferredPort));
while (claimedPorts.has(port)) {
port += 1;
}
return port;
}
function buildIsolatedWorktreeConfig(
config: PaperclipConfig,
context: WorktreeRuntimeContext,
portOverrides?: {
serverPort?: number;
databasePort?: number;
},
): PaperclipConfig {
const serverPort = portOverrides?.serverPort ?? config.server.port;
const databasePort =
config.database.mode === "embedded-postgres"
? portOverrides?.databasePort ?? config.database.embeddedPostgresPort
: undefined;
const nextConfig: PaperclipConfig = {
...config,
database: {
...config.database,
...(config.database.mode === "embedded-postgres"
? {
embeddedPostgresDataDir: context.embeddedPostgresDataDir,
embeddedPostgresPort: databasePort ?? config.database.embeddedPostgresPort,
backup: {
...config.database.backup,
enabled: false,
dir: context.backupDir,
},
}
: {}),
},
server: {
...config.server,
port: serverPort,
},
logging: {
...config.logging,
logDir: context.logDir,
},
storage: {
...config.storage,
localDisk: {
...config.storage.localDisk,
baseDir: context.storageDir,
},
},
secrets: {
...config.secrets,
localEncrypted: {
...config.secrets.localEncrypted,
keyFilePath: context.secretsKeyFilePath,
},
},
};
if (config.auth.baseUrlMode === "explicit" && config.auth.publicBaseUrl) {
nextConfig.auth = {
...config.auth,
publicBaseUrl: rewriteLocalUrlPort(config.auth.publicBaseUrl, serverPort),
};
}
return nextConfig;
}
function needsWorktreeConfigRepair(
config: PaperclipConfig,
context: WorktreeRuntimeContext,
): boolean {
if (config.database.mode === "embedded-postgres") {
if (config.database.backup.enabled) {
return true;
}
if (!isPathInside(config.database.embeddedPostgresDataDir, context.instanceRoot)) {
return true;
}
if (!isPathInside(config.database.backup.dir, context.instanceRoot)) {
return true;
}
}
if (!isPathInside(config.logging.logDir, context.instanceRoot)) {
return true;
}
if (!isPathInside(config.storage.localDisk.baseDir, context.instanceRoot)) {
return true;
}
if (!isPathInside(config.secrets.localEncrypted.keyFilePath, context.instanceRoot)) {
return true;
}
return false;
}
export function applyRuntimePortSelectionToConfig(
config: PaperclipConfig,
input: {
serverPort: number;
databasePort?: number | null;
allowServerPortWrite?: boolean;
allowDatabasePortWrite?: boolean;
},
): { config: PaperclipConfig; changed: boolean } {
let changed = false;
let nextConfig = config;
if (input.allowServerPortWrite !== false && config.server.port !== input.serverPort) {
nextConfig = {
...nextConfig,
server: {
...nextConfig.server,
port: input.serverPort,
},
};
changed = true;
}
if (
input.allowDatabasePortWrite !== false &&
nextConfig.database.mode === "embedded-postgres" &&
typeof input.databasePort === "number" &&
nextConfig.database.embeddedPostgresPort !== input.databasePort
) {
nextConfig = {
...nextConfig,
database: {
...nextConfig.database,
embeddedPostgresPort: input.databasePort,
},
};
changed = true;
}
if (nextConfig.auth.baseUrlMode === "explicit" && nextConfig.auth.publicBaseUrl) {
const rewritten = rewriteLocalUrlPort(nextConfig.auth.publicBaseUrl, input.serverPort);
if (rewritten && rewritten !== nextConfig.auth.publicBaseUrl) {
nextConfig = {
...nextConfig,
auth: {
...nextConfig.auth,
publicBaseUrl: rewritten,
},
};
changed = true;
}
}
return { config: nextConfig, changed };
}
export function maybeRepairLegacyWorktreeConfigAndEnvFiles(): {
repairedConfig: boolean;
repairedEnv: boolean;
} {
const context = resolveWorktreeRuntimeContext(process.env);
if (!context) {
return { repairedConfig: false, repairedEnv: false };
}
process.env.PAPERCLIP_HOME = context.homeDir;
process.env.PAPERCLIP_INSTANCE_ID = context.instanceId;
process.env.PAPERCLIP_CONFIG = context.configPath;
process.env.PAPERCLIP_CONTEXT = context.contextPath;
process.env.PAPERCLIP_WORKTREE_NAME = context.worktreeName;
let repairedConfig = false;
if (fs.existsSync(context.configPath)) {
try {
const runtimeConfig = withWorktreePortRegistryLock(context.homeDir, () => {
const parsed = JSON.parse(fs.readFileSync(context.configPath, "utf8")) as PaperclipConfig;
let selectedConfig = parsed;
const registeredConfigPaths = readWorktreePortRegistry(context.homeDir);
const siblingPorts = collectSiblingWorktreePorts(context, registeredConfigPaths);
const serverPortCollision = siblingPorts.serverPorts.has(parsed.server.port);
const databasePortCollision =
parsed.database.mode === "embedded-postgres" &&
siblingPorts.databasePorts.has(parsed.database.embeddedPostgresPort);
if (needsWorktreeConfigRepair(parsed, context) || serverPortCollision || databasePortCollision) {
const selectedServerPort = findNextUnclaimedPort(
parsed.server.port === 3100 ? 3101 : parsed.server.port,
siblingPorts.serverPorts,
);
const selectedDatabasePort =
parsed.database.mode === "embedded-postgres"
? findNextUnclaimedPort(
parsed.database.embeddedPostgresPort === 54329
? 54330
: parsed.database.embeddedPostgresPort,
new Set([...siblingPorts.databasePorts, selectedServerPort]),
)
: undefined;
selectedConfig = buildIsolatedWorktreeConfig(parsed, context, {
serverPort: selectedServerPort,
databasePort: selectedDatabasePort,
});
repairedConfig = writeConfigFile(context.configPath, selectedConfig);
if (serverPortCollision || databasePortCollision) {
console.warn(
[
`Worktree port conflict detected for ${context.worktreeName}; updated and persisted workspace ports.`,
...(serverPortCollision
? [`server: ${parsed.server.port} -> ${selectedServerPort}`]
: []),
...(databasePortCollision && parsed.database.mode === "embedded-postgres"
? [`database: ${parsed.database.embeddedPostgresPort} -> ${selectedDatabasePort}`]
: []),
].join(" "),
);
}
}
writeWorktreePortRegistry(context.homeDir, [
...registeredConfigPaths,
...siblingPorts.configPaths,
context.configPath,
]);
return selectedConfig;
});
if (
!nonEmpty(process.env.PORT)
&& Number.isInteger(runtimeConfig.server.port)
&& runtimeConfig.server.port > 0
) {
process.env.PORT = String(runtimeConfig.server.port);
}
} catch {
// Leave invalid configs to the normal startup validation path.
}
}
const existingEnvEntries = readEnvEntries(context.envPath);
const managedEnvEntries: Record<string, string> = {
PAPERCLIP_HOME: context.homeDir,
PAPERCLIP_INSTANCE_ID: context.instanceId,
PAPERCLIP_CONFIG: context.configPath,
PAPERCLIP_CONTEXT: context.contextPath,
PAPERCLIP_IN_WORKTREE: "true",
PAPERCLIP_DB_BACKUP_ENABLED: "false",
PAPERCLIP_WORKTREE_NAME: context.worktreeName,
};
process.env.PAPERCLIP_DB_BACKUP_ENABLED = "false";
const repairedEnv = Object.entries(managedEnvEntries).some(
([key, value]) => existingEnvEntries[key] !== value,
);
if (repairedEnv) {
fs.mkdirSync(path.dirname(context.envPath), { recursive: true });
const existingContents = fs.existsSync(context.envPath)
? fs.readFileSync(context.envPath, "utf8")
: null;
const repairedContents =
existingContents === null
? formatEnvEntries(managedEnvEntries)
: updateEnvFileContents(existingContents, managedEnvEntries);
fs.writeFileSync(context.envPath, repairedContents, { mode: 0o600 });
}
return { repairedConfig, repairedEnv };
}
export function maybePersistWorktreeRuntimePorts(input: {
serverPort: number;
databasePort?: number | null;
}): void {
const context = resolveWorktreeRuntimeContext(process.env);
if (!context || !fs.existsSync(context.configPath)) return;
let fileConfig: PaperclipConfig;
try {
fileConfig = JSON.parse(fs.readFileSync(context.configPath, "utf8")) as PaperclipConfig;
} catch {
return;
}
const { config, changed } = applyRuntimePortSelectionToConfig(fileConfig, {
serverPort: input.serverPort,
databasePort: input.databasePort,
allowServerPortWrite: !nonEmpty(process.env.PORT),
allowDatabasePortWrite: !nonEmpty(process.env.DATABASE_URL),
});
if (changed) {
writeConfigFile(context.configPath, config);
}
}