diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 47652d17b0a..c2238b20f14 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -75,6 +75,17 @@ pnpm dev:list pnpm dev:stop ``` +When embedded PostgreSQL is used, server startup also performs a conservative +stale-runtime cleanup before opening the database. It only targets unregistered +Node-family runtime processes with Paperclip ownership evidence: an allowlisted +Paperclip runtime env marker or a known Paperclip server/dev-runner entrypoint, +plus cwd or command-line containment under the current Paperclip repo/worktree +or instance roots. It skips the current process group plus registered runtime +services. This is the automatic repair path for worktree-local `too many +clients` failures caused by orphaned Paperclip dev servers holding embedded +Postgres connections. Set +`PAPERCLIP_STALE_RUNTIME_CLEANUP=false` to disable this startup repair. + `pnpm dev:once` now tracks backend-relevant file changes and pending migrations. When the current boot is stale, the board UI shows a `Restart required` banner. You can also enable guarded auto-restart in `Instance Settings > Experimental`, which waits for queued/running local agent runs to finish before restarting the dev server. Tailscale/private-auth dev mode: diff --git a/scripts/dev-runner.ts b/scripts/dev-runner.ts index ba7fcb4cda4..40332366fc6 100644 --- a/scripts/dev-runner.ts +++ b/scripts/dev-runner.ts @@ -189,6 +189,9 @@ const devService = createDevServiceIdentity({ networkProfile: tailscaleAuth ? `legacy:${bindMode ?? "lan"}` : (bindMode ?? "default"), port: serverPort, }); +env.PAPERCLIP_MANAGED_RUNTIME = "paperclip-dev"; +env.PAPERCLIP_LOCAL_SERVICE_KEY = devService.serviceKey; +env.PAPERCLIP_RUNTIME_SERVICE_NAME = devService.serviceName; const existingRunner = await findAdoptableLocalService({ serviceKey: devService.serviceKey, diff --git a/server/src/__tests__/local-service-supervisor.test.ts b/server/src/__tests__/local-service-supervisor.test.ts new file mode 100644 index 00000000000..c28e3ec4590 --- /dev/null +++ b/server/src/__tests__/local-service-supervisor.test.ts @@ -0,0 +1,296 @@ +import { describe, expect, it } from "vitest"; +import { + cleanupOrphanedPaperclipRuntimeProcesses, + selectOrphanedPaperclipRuntimeProcesses, + type LocalProcessSnapshot, +} from "../services/local-service-supervisor.ts"; + +function proc(partial: Partial & { pid: number }): LocalProcessSnapshot { + return { + parentPid: null, + processGroupId: null, + commandName: "node", + commandLine: "node server/src/index.ts", + cwd: null, + ...partial, + }; +} + +describe("local service stale runtime cleanup", () => { + it("detects unregistered Paperclip-marked dev processes contained by a workspace root", () => { + const root = "/tmp/paperclip-worktree/PAP-11663-cleanup"; + const candidates = selectOrphanedPaperclipRuntimeProcesses({ + containmentRoots: [root], + currentProcessId: 1, + currentProcessGroupId: 10, + processes: [ + proc({ + pid: 1200, + processGroupId: 2200, + commandName: "pnpm", + commandLine: "pnpm dev", + cwd: root, + paperclipRuntimeMarkers: { + PAPERCLIP_MANAGED_RUNTIME: "workspace-runtime", + }, + }), + ], + }); + + expect(candidates).toHaveLength(1); + expect(candidates[0]).toMatchObject({ + pid: 1200, + processGroupId: 2200, + matchedRoot: root, + containmentEvidence: "cwd", + ownershipEvidence: "paperclip_runtime_env", + }); + }); + + it("accepts command-line containment only for path tokens under the root", () => { + const root = "/tmp/paperclip"; + const candidates = selectOrphanedPaperclipRuntimeProcesses({ + containmentRoots: [root], + currentProcessId: 1, + currentProcessGroupId: 10, + processes: [ + proc({ + pid: 1200, + processGroupId: 2200, + commandLine: `node ${root}/server/src/index.ts`, + cwd: null, + }), + proc({ + pid: 1201, + processGroupId: 2201, + commandName: "vite", + commandLine: `vite --root=${root}/apps/board --host 127.0.0.1`, + cwd: null, + paperclipRuntimeMarkers: { + PAPERCLIP_MANAGED_RUNTIME: "workspace-runtime", + }, + }), + ], + }); + + expect(candidates).toHaveLength(2); + expect(candidates.map((candidate) => candidate.containmentEvidence)).toEqual(["command", "command"]); + expect(candidates.map((candidate) => candidate.ownershipEvidence)).toEqual([ + "paperclip_entrypoint", + "paperclip_runtime_env", + ]); + }); + + it("ignores processes without containment, non-node dev intent, managed pids, and the current process group", () => { + const root = "/tmp/paperclip-worktree/PAP-11663-cleanup"; + const candidates = selectOrphanedPaperclipRuntimeProcesses({ + containmentRoots: [root], + managedProcessIds: [1400, 2400], + currentProcessId: 1, + currentProcessGroupId: 99, + processes: [ + proc({ + pid: 1200, + processGroupId: 2200, + commandLine: "node /tmp/other/server/src/index.ts", + cwd: "/tmp/other", + }), + proc({ + pid: 1300, + processGroupId: 2300, + commandName: "python", + commandLine: "python -m http.server", + cwd: root, + paperclipRuntimeMarkers: { + PAPERCLIP_MANAGED_RUNTIME: "workspace-runtime", + }, + }), + proc({ + pid: 1400, + processGroupId: 2400, + commandLine: `node ${root}/server/src/index.ts`, + cwd: root, + }), + proc({ + pid: 1500, + processGroupId: 99, + commandLine: `node ${root}/server/src/index.ts`, + cwd: root, + }), + ], + }); + + expect(candidates).toEqual([]); + }); + + it("does not select unrelated nested package dev processes under known roots", () => { + const root = "/tmp/paperclip-worktree/PAP-11663-cleanup"; + const candidates = selectOrphanedPaperclipRuntimeProcesses({ + containmentRoots: [root], + currentProcessId: 1, + currentProcessGroupId: 10, + processes: [ + proc({ + pid: 1200, + processGroupId: 2200, + commandName: "pnpm", + commandLine: "pnpm dev", + cwd: `${root}/unrelated-app`, + }), + proc({ + pid: 1201, + processGroupId: 2201, + commandName: "node", + commandLine: `node ${root}/unrelated-app/node_modules/vite/bin/vite.js --host 127.0.0.1`, + cwd: `${root}/unrelated-app`, + }), + ], + }); + + expect(candidates).toEqual([]); + }); + + it("does not treat sibling path prefixes as command-line containment", () => { + const candidates = selectOrphanedPaperclipRuntimeProcesses({ + containmentRoots: ["/tmp/paperclip"], + currentProcessId: 1, + currentProcessGroupId: 10, + processes: [ + proc({ + pid: 1234, + parentPid: null, + processGroupId: 2222, + commandName: "node", + commandLine: "node /tmp/paperclip-evil/server/src/index.ts", + cwd: "/tmp/elsewhere", + }), + ], + }); + + expect(candidates).toEqual([]); + }); + + it("does not classify one-shot build tooling as stale runtime servers", () => { + const root = "/tmp/paperclip-worktree/PAP-11663-cleanup"; + const candidates = selectOrphanedPaperclipRuntimeProcesses({ + containmentRoots: [root], + currentProcessId: 1, + currentProcessGroupId: 10, + processes: [ + proc({ + pid: 1200, + processGroupId: 2200, + commandName: "node", + commandLine: `node ${root}/node_modules/vite/bin/vite.js build`, + cwd: root, + }), + proc({ + pid: 1201, + processGroupId: 2201, + commandName: "esbuild", + commandLine: `esbuild ${root}/src/index.ts --bundle --outfile=${root}/dist/index.js`, + cwd: root, + }), + proc({ + pid: 1202, + processGroupId: 2202, + commandName: "tsx", + commandLine: `tsx ${root}/scripts/one-shot-maintenance.ts`, + cwd: root, + }), + ], + }); + + expect(candidates).toEqual([]); + }); + + it("honors registry metadata child pids as managed runtime processes", async () => { + const root = "/tmp/paperclip-worktree/PAP-11663-cleanup"; + const result = await cleanupOrphanedPaperclipRuntimeProcesses({ + containmentRoots: [root], + currentProcessId: 1, + currentProcessGroupId: 10, + registryRecords: [ + { + version: 1, + serviceKey: "paperclip-dev-watch", + profileKind: "paperclip-dev", + serviceName: "paperclip-dev-watch", + command: "dev-runner.ts", + cwd: root, + envFingerprint: "fingerprint", + port: 3100, + url: "http://127.0.0.1:3100", + pid: 1200, + processGroupId: null, + provider: "local_process", + runtimeServiceId: null, + reuseKey: null, + startedAt: new Date(0).toISOString(), + lastSeenAt: new Date(0).toISOString(), + metadata: { + childPid: 1201, + }, + }, + ], + processSnapshots: [ + proc({ + pid: 1201, + processGroupId: 2200, + commandLine: `node ${root}/server/src/index.ts`, + cwd: root, + }), + ], + terminate: async () => { + throw new Error("managed child pid should not be terminated"); + }, + }); + + expect(result).toMatchObject({ + scanned: 1, + matched: 0, + terminated: 0, + }); + }); + + it("terminates each orphaned process group once", async () => { + const root = "/tmp/paperclip-worktree/PAP-11663-cleanup"; + const terminated: Array<{ pid: number; processGroupId: number | null }> = []; + + const result = await cleanupOrphanedPaperclipRuntimeProcesses({ + containmentRoots: [root], + currentProcessId: 1, + currentProcessGroupId: 10, + registryRecords: [], + processSnapshots: [ + proc({ + pid: 1200, + processGroupId: 2200, + commandLine: `node ${root}/server/src/index.ts`, + cwd: root, + }), + proc({ + pid: 1201, + processGroupId: 2200, + commandName: "esbuild", + commandLine: `esbuild --service=0.28.0 ${root}`, + cwd: root, + paperclipRuntimeMarkers: { + PAPERCLIP_MANAGED_RUNTIME: "workspace-runtime", + }, + }), + ], + terminate: async (record) => { + terminated.push({ pid: record.pid, processGroupId: record.processGroupId }); + }, + }); + + expect(result).toMatchObject({ + scanned: 2, + matched: 2, + terminated: 1, + dryRun: false, + }); + expect(terminated).toEqual([{ pid: 1200, processGroupId: 2200 }]); + }); +}); diff --git a/server/src/index.ts b/server/src/index.ts index 88894a204f5..22375bdac57 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -6,7 +6,7 @@ import { instrumentationReady, shutdownInstrumentation } from "./instrumentation.js"; import { existsSync, readFileSync, rmSync } from "node:fs"; import { createServer } from "node:http"; -import { resolve } from "node:path"; +import { basename, dirname, resolve } from "node:path"; import { createInterface } from "node:readline/promises"; import { stdin, stdout } from "node:process"; import { pathToFileURL } from "node:url"; @@ -31,7 +31,7 @@ import { } from "@paperclipai/db"; import detectPort from "detect-port"; import { createApp } from "./app.js"; -import { loadConfig } from "./config.js"; +import { loadConfig, type Config } from "./config.js"; import { logger } from "./middleware/logger.js"; import { setupEnvironmentCustomImageTerminalWebSocketServer } from "./realtime/environment-custom-image-terminal-ws.js"; import { setupLiveEventsWebSocketServer } from "./realtime/live-events-ws.js"; @@ -54,12 +54,14 @@ import { import { createFeedbackTraceShareClientFromConfig } from "./services/feedback-share-client.js"; import { buildRuntimeApiCandidateUrls, choosePrimaryRuntimeApiUrl } from "./runtime-api.js"; import { createPluginWorkerManager } from "./services/plugin-worker-manager.js"; +import { cleanupOrphanedPaperclipRuntimeProcesses } from "./services/local-service-supervisor.js"; import { createStorageServiceFromConfig } from "./storage/index.js"; import { printStartupBanner } from "./startup-banner.js"; import { getBoardClaimWarningUrl, initializeBoardClaimChallenge } from "./board-claim.js"; import { maybePersistWorktreeRuntimePorts } from "./worktree-config.js"; import { initTelemetry, getTelemetryClient } from "./telemetry.js"; import { conflict } from "./errors.js"; +import { resolvePaperclipConfigPath } from "./paths.js"; import type { InstanceDatabaseBackupRunResult, InstanceDatabaseBackupTrigger, @@ -102,6 +104,48 @@ export interface StartedServer { databaseUrl: string; } +function collectPreDatabaseRuntimeCleanupRoots(config: Config): string[] { + const roots = new Set(); + roots.add(process.cwd()); + roots.add(dirname(resolve(config.embeddedPostgresDataDir))); + + const configPath = resolvePaperclipConfigPath(); + const configDir = dirname(configPath); + roots.add(configDir); + if (basename(configDir) === ".paperclip") { + roots.add(resolve(configDir, "..")); + } + + return Array.from(roots); +} + +async function cleanupStaleRuntimeProcessesBeforeDatabaseStartup(config: Config) { + if (config.databaseUrl) return; + if (process.env.PAPERCLIP_STALE_RUNTIME_CLEANUP === "false") return; + + const result = await cleanupOrphanedPaperclipRuntimeProcesses({ + containmentRoots: collectPreDatabaseRuntimeCleanupRoots(config), + }); + if (result.matched === 0) return; + + logger.warn( + { + matched: result.matched, + terminated: result.terminated, + candidates: result.candidates.map((candidate) => ({ + pid: candidate.pid, + processGroupId: candidate.processGroupId, + commandName: candidate.commandName, + cwd: candidate.cwd, + matchedRoot: candidate.matchedRoot, + containmentEvidence: candidate.containmentEvidence, + ownershipEvidence: candidate.ownershipEvidence, + })), + }, + "Cleaned stale Paperclip-owned runtime processes before embedded PostgreSQL startup", + ); +} + export async function startServer(): Promise { // Tracing must be active (or have failed and logged) before the first DB // connection or the HTTP server exists — see instrumentation.ts. @@ -117,7 +161,8 @@ export async function startServer(): Promise { if (process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE === undefined) { process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = config.secretsMasterKeyFilePath; } - + await cleanupStaleRuntimeProcessesBeforeDatabaseStartup(config); + type MigrationSummary = | "skipped" | "already applied" @@ -700,7 +745,7 @@ export async function startServer(): Promise { process.env.PAPERCLIP_RUNTIME_API_URL = runtimeApiUrl; process.env.PAPERCLIP_RUNTIME_API_CANDIDATES_JSON = JSON.stringify(runtimeApiCandidates); process.env.PAPERCLIP_API_URL = configuredApiUrl; - + setupEnvironmentCustomImageTerminalWebSocketServer(server, db as any, { pluginWorkerManager, }); @@ -713,10 +758,28 @@ export async function startServer(): Promise { .then((result) => { if (result.reconciled > 0) { logger.warn( - { reconciled: result.reconciled }, + { reconciled: result.reconciled, adopted: result.adopted, stopped: result.stopped }, "reconciled persisted runtime services from a previous server process", ); } + if (result.orphanedProcessesCleaned > 0) { + logger.warn( + { + matched: result.orphanedProcessesMatched, + cleaned: result.orphanedProcessesCleaned, + candidates: result.orphanedProcessCandidates.map((candidate) => ({ + pid: candidate.pid, + processGroupId: candidate.processGroupId, + commandName: candidate.commandName, + cwd: candidate.cwd, + matchedRoot: candidate.matchedRoot, + containmentEvidence: candidate.containmentEvidence, + ownershipEvidence: candidate.ownershipEvidence, + })), + }, + "cleaned stale unregistered workspace runtime processes during startup reconciliation", + ); + } }) .catch((err) => { logger.error({ err }, "startup reconciliation of persisted runtime services failed"); diff --git a/server/src/services/local-service-supervisor.ts b/server/src/services/local-service-supervisor.ts index 86d270e37ad..8ba4747f0af 100644 --- a/server/src/services/local-service-supervisor.ts +++ b/server/src/services/local-service-supervisor.ts @@ -1,12 +1,27 @@ import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import fs from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { promisify } from "node:util"; import { resolvePaperclipInstanceRoot } from "../home-paths.js"; const execFileAsync = promisify(execFile); +const PAPERCLIP_RUNTIME_MARKER_KEYS = [ + "PAPERCLIP_MANAGED_RUNTIME", + "PAPERCLIP_LOCAL_SERVICE_KEY", + "PAPERCLIP_RUNTIME_SERVICE_ID", + "PAPERCLIP_RUNTIME_SERVICE_NAME", +] as const; +const PAPERCLIP_RUNTIME_MARKER_KEY_SET = new Set(PAPERCLIP_RUNTIME_MARKER_KEYS); +const PAPERCLIP_RUNTIME_ENTRYPOINTS = new Set([ + "scripts/dev-runner.ts", + "server/scripts/dev-watch.ts", + "server/src/index.ts", +]); + +export type PaperclipRuntimeMarkers = Partial>; export interface LocalServiceRegistryRecord { version: 1; @@ -38,6 +53,35 @@ export interface LocalServiceIdentityInput { scope: Record | null; } +export interface LocalProcessSnapshot { + pid: number; + parentPid: number | null; + processGroupId: number | null; + commandName: string; + commandLine: string; + cwd: string | null; + paperclipRuntimeMarkers?: PaperclipRuntimeMarkers | null; +} + +export interface OrphanedPaperclipRuntimeProcess { + pid: number; + processGroupId: number | null; + commandName: string; + commandLine: string; + cwd: string | null; + matchedRoot: string; + containmentEvidence: "cwd" | "command"; + ownershipEvidence: "paperclip_runtime_env" | "paperclip_entrypoint"; +} + +export interface OrphanedPaperclipRuntimeCleanupResult { + scanned: number; + matched: number; + terminated: number; + dryRun: boolean; + candidates: OrphanedPaperclipRuntimeProcess[]; +} + function stableStringify(value: unknown): string { if (Array.isArray(value)) { return `[${value.map((entry) => stableStringify(entry)).join(",")}]`; @@ -49,6 +93,236 @@ function stableStringify(value: unknown): string { return JSON.stringify(value); } +function normalizeWhitespace(value: string) { + return value.replace(/\s+/g, " ").trim(); +} + +function normalizeProcessPath(value: string) { + return path.resolve(value); +} + +function isUsableContainmentRoot(root: string) { + const parsed = path.parse(root); + if (root === parsed.root) return false; + if (root === os.homedir()) return false; + return root.length >= parsed.root.length + 6; +} + +function isPathContainedByRoot(candidatePath: string | null | undefined, root: string) { + if (!candidatePath) return false; + const relative = path.relative(root, normalizeProcessPath(candidatePath)); + return relative === "" || (!!relative && !relative.startsWith("..") && !path.isAbsolute(relative)); +} + +function tokenizeCommandLine(commandLine: string) { + const tokens: string[] = []; + let current = ""; + let quote: "'" | "\"" | null = null; + let escaped = false; + + for (const char of commandLine) { + if (escaped) { + current += char; + escaped = false; + continue; + } + if (char === "\\") { + escaped = true; + continue; + } + if (quote) { + if (char === quote) { + quote = null; + } else { + current += char; + } + continue; + } + if (char === "'" || char === "\"") { + quote = char; + continue; + } + if (/\s/.test(char)) { + if (current) { + tokens.push(current); + current = ""; + } + continue; + } + current += char; + } + + if (escaped) current += "\\"; + if (current) tokens.push(current); + return tokens; +} + +function cleanCommandLinePathToken(token: string) { + return token.replace(/[,;]+$/g, ""); +} + +function commandLineHasContainedPath(commandLine: string, root: string) { + if (!commandLine || !root) return false; + for (const token of tokenizeCommandLine(commandLine)) { + const candidates = [token]; + const equalsIndex = token.indexOf("="); + if (equalsIndex >= 0) candidates.push(token.slice(equalsIndex + 1)); + + for (const candidate of candidates) { + const cleaned = cleanCommandLinePathToken(candidate); + if (!path.isAbsolute(cleaned)) continue; + if (isPathContainedByRoot(cleaned, root)) return true; + } + } + return false; +} + +function looksLikeNodeRuntimeProcess(snapshot: LocalProcessSnapshot) { + const commandName = path.basename(snapshot.commandName || "").toLowerCase(); + const commandLine = normalizeWhitespace(snapshot.commandLine).toLowerCase(); + + return ( + commandName === "node" || + commandName === "tsx" || + commandName === "esbuild" || + commandName === "vite" || + commandName === "next" || + commandName === "astro" || + commandName === "webpack" || + commandName === "turbo" || + commandName === "pnpm" || + commandName === "npm" || + commandName === "yarn" || + commandName === "bun" || + /(^|[/\s])(?:node|tsx|esbuild|vite|next|astro|webpack|turbo|pnpm|npm|yarn|bun)(?:$|[\s/'"])/.test(commandLine) + ); +} + +function hasPaperclipRuntimeEnvMarker(snapshot: LocalProcessSnapshot) { + const markers = snapshot.paperclipRuntimeMarkers; + if (!markers) return false; + return PAPERCLIP_RUNTIME_MARKER_KEYS.some((key) => typeof markers[key] === "string" && markers[key]!.trim().length > 0); +} + +function resolveCommandLinePathToken(token: string, cwd: string | null | undefined) { + const cleaned = cleanCommandLinePathToken(token); + if (!cleaned) return null; + if (path.isAbsolute(cleaned)) return normalizeProcessPath(cleaned); + if (!cwd || !/[\\/]/.test(cleaned)) return null; + return normalizeProcessPath(path.resolve(cwd, cleaned)); +} + +function commandLineHasPaperclipEntrypoint(commandLine: string, root: string, cwd: string | null | undefined) { + if (!commandLine || !root) return false; + for (const token of tokenizeCommandLine(commandLine)) { + const candidates = [token]; + const equalsIndex = token.indexOf("="); + if (equalsIndex >= 0) candidates.push(token.slice(equalsIndex + 1)); + + for (const candidate of candidates) { + const resolved = resolveCommandLinePathToken(candidate, cwd); + if (!resolved || !isPathContainedByRoot(resolved, root)) continue; + const relative = path.relative(root, resolved).split(path.sep).join("/"); + if (PAPERCLIP_RUNTIME_ENTRYPOINTS.has(relative)) return true; + } + } + return false; +} + +function getPaperclipOwnershipEvidence(snapshot: LocalProcessSnapshot, root: string) { + if (hasPaperclipRuntimeEnvMarker(snapshot)) return "paperclip_runtime_env" as const; + if (commandLineHasPaperclipEntrypoint(snapshot.commandLine, root, snapshot.cwd)) return "paperclip_entrypoint" as const; + return null; +} + +function normalizeManagedProcessIds(input?: Iterable) { + const ids = new Set(); + for (const value of input ?? []) { + if (typeof value === "number" && Number.isInteger(value) && value > 0) { + ids.add(value); + } + } + return ids; +} + +function collectManagedProcessIdsFromRegistry(records: LocalServiceRegistryRecord[]) { + const ids = new Set(); + for (const record of records) { + ids.add(record.pid); + if (record.processGroupId) ids.add(record.processGroupId); + const childPid = record.metadata?.childPid; + if (typeof childPid === "number" && Number.isInteger(childPid) && childPid > 0) { + ids.add(childPid); + } + } + return ids; +} + +export function selectOrphanedPaperclipRuntimeProcesses(input: { + processes: LocalProcessSnapshot[]; + containmentRoots: string[]; + managedProcessIds?: Iterable; + currentProcessId?: number; + currentProcessGroupId?: number | null; +}): OrphanedPaperclipRuntimeProcess[] { + const roots = Array.from( + new Set( + input.containmentRoots + .map((root) => normalizeProcessPath(root)) + .filter(isUsableContainmentRoot), + ), + ).sort((left, right) => right.length - left.length); + if (roots.length === 0) return []; + + const managedProcessIds = normalizeManagedProcessIds(input.managedProcessIds); + const currentProcessId = input.currentProcessId ?? process.pid; + const currentProcessGroupId = input.currentProcessGroupId ?? null; + const selected: OrphanedPaperclipRuntimeProcess[] = []; + + for (const processSnapshot of input.processes) { + if (!Number.isInteger(processSnapshot.pid) || processSnapshot.pid <= 0) continue; + if (processSnapshot.pid === currentProcessId) continue; + if (managedProcessIds.has(processSnapshot.pid)) continue; + if (processSnapshot.processGroupId && managedProcessIds.has(processSnapshot.processGroupId)) continue; + if (currentProcessGroupId && processSnapshot.processGroupId === currentProcessGroupId) continue; + if (!looksLikeNodeRuntimeProcess(processSnapshot)) continue; + + let matchedRoot: string | null = null; + let containmentEvidence: "cwd" | "command" | null = null; + let ownershipEvidence: OrphanedPaperclipRuntimeProcess["ownershipEvidence"] | null = null; + for (const root of roots) { + const evidence = getPaperclipOwnershipEvidence(processSnapshot, root); + if (!evidence) continue; + if (isPathContainedByRoot(processSnapshot.cwd, root)) { + matchedRoot = root; + containmentEvidence = "cwd"; + ownershipEvidence = evidence; + break; + } + if (commandLineHasContainedPath(processSnapshot.commandLine, root)) { + matchedRoot = root; + containmentEvidence = "command"; + ownershipEvidence = evidence; + break; + } + } + if (!matchedRoot || !containmentEvidence || !ownershipEvidence) continue; + + selected.push({ + pid: processSnapshot.pid, + processGroupId: processSnapshot.processGroupId, + commandName: processSnapshot.commandName, + commandLine: normalizeWhitespace(processSnapshot.commandLine), + cwd: processSnapshot.cwd, + matchedRoot, + containmentEvidence, + ownershipEvidence, + }); + } + + return selected.sort((left, right) => left.pid - right.pid); +} + function sanitizeServiceKeySegment(value: string, fallback: string): string { const normalized = value .trim() @@ -177,6 +451,172 @@ export async function listLocalServiceRegistryRecords(filter?: { } } +// Cap concurrent /proc reads so a machine with hundreds of processes does not +// issue hundreds of simultaneous filesystem reads at startup (this scan runs +// twice: pre-database cleanup and startup reconciliation). +const LOCAL_PROCESS_SNAPSHOT_READ_CONCURRENCY = 24; + +async function mapWithConcurrencyLimit( + items: readonly T[], + limit: number, + mapper: (item: T, index: number) => Promise, +): Promise { + if (items.length === 0) return []; + const effectiveLimit = Math.max(1, Math.min(limit, items.length)); + const results = new Array(items.length); + let nextIndex = 0; + async function worker() { + for (;;) { + const current = nextIndex; + nextIndex += 1; + if (current >= items.length) return; + results[current] = await mapper(items[current]!, current); + } + } + await Promise.all(Array.from({ length: effectiveLimit }, () => worker())); + return results; +} + +async function readProcessCwd(pid: number) { + if (process.platform !== "linux") return null; + try { + return await fs.realpath(`/proc/${pid}/cwd`); + } catch { + return null; + } +} + +async function readProcessPaperclipRuntimeMarkers(pid: number): Promise { + if (process.platform !== "linux") return null; + try { + const raw = await fs.readFile(`/proc/${pid}/environ`, "utf8"); + const markers: PaperclipRuntimeMarkers = {}; + for (const entry of raw.split("\0")) { + const equalsIndex = entry.indexOf("="); + if (equalsIndex <= 0) continue; + const key = entry.slice(0, equalsIndex); + if (!PAPERCLIP_RUNTIME_MARKER_KEY_SET.has(key)) continue; + const value = entry.slice(equalsIndex + 1); + if (value.trim().length > 0) { + markers[key as keyof PaperclipRuntimeMarkers] = value; + } + } + return Object.keys(markers).length > 0 ? markers : null; + } catch { + return null; + } +} + +function parsePsProcessLine(line: string): LocalProcessSnapshot | null { + const match = line.trim().match(/^(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s*(.*)$/); + if (!match) return null; + const pid = Number.parseInt(match[1]!, 10); + const parentPid = Number.parseInt(match[2]!, 10); + const processGroupId = Number.parseInt(match[3]!, 10); + if (!Number.isInteger(pid) || pid <= 0) return null; + return { + pid, + parentPid: Number.isInteger(parentPid) && parentPid > 0 ? parentPid : null, + processGroupId: Number.isInteger(processGroupId) && processGroupId > 0 ? processGroupId : null, + commandName: match[4] ?? "", + commandLine: match[5]?.trim() || match[4] || "", + cwd: null, + }; +} + +export async function listLocalProcessSnapshots(): Promise { + if (process.platform === "win32") return []; + try { + const { stdout } = await execFileAsync("ps", ["-axo", "pid=,ppid=,pgid=,comm=,args="]); + const parsed = stdout + .split(/\r?\n/) + .map(parsePsProcessLine) + .filter((entry): entry is LocalProcessSnapshot => entry !== null); + return await mapWithConcurrencyLimit( + parsed, + LOCAL_PROCESS_SNAPSHOT_READ_CONCURRENCY, + async (entry) => ({ + ...entry, + cwd: await readProcessCwd(entry.pid), + paperclipRuntimeMarkers: await readProcessPaperclipRuntimeMarkers(entry.pid), + }), + ); + } catch { + return []; + } +} + +async function readCurrentProcessGroupId() { + if (process.platform === "win32") return null; + try { + const { stdout } = await execFileAsync("ps", ["-o", "pgid=", "-p", String(process.pid)]); + const parsed = Number.parseInt(stdout.trim(), 10); + return Number.isInteger(parsed) && parsed > 0 ? parsed : null; + } catch { + return null; + } +} + +export async function cleanupOrphanedPaperclipRuntimeProcesses(input: { + containmentRoots: string[]; + dryRun?: boolean; + additionalManagedProcessIds?: Iterable; + processSnapshots?: LocalProcessSnapshot[]; + registryRecords?: LocalServiceRegistryRecord[]; + currentProcessId?: number; + currentProcessGroupId?: number | null; + terminate?: (record: Pick) => Promise; +}): Promise { + const processSnapshots = input.processSnapshots ?? await listLocalProcessSnapshots(); + const registryRecords = input.registryRecords ?? await listLocalServiceRegistryRecords(); + const managedProcessIds = collectManagedProcessIdsFromRegistry(registryRecords); + for (const id of input.additionalManagedProcessIds ?? []) { + if (typeof id === "number" && Number.isInteger(id) && id > 0) { + managedProcessIds.add(id); + } + } + const currentProcessGroupId = input.currentProcessGroupId ?? await readCurrentProcessGroupId(); + const candidates = selectOrphanedPaperclipRuntimeProcesses({ + processes: processSnapshots, + containmentRoots: input.containmentRoots, + managedProcessIds, + currentProcessId: input.currentProcessId, + currentProcessGroupId, + }); + + if (input.dryRun) { + return { + scanned: processSnapshots.length, + matched: candidates.length, + terminated: 0, + dryRun: true, + candidates, + }; + } + + const terminate = input.terminate ?? ((record) => terminateLocalService(record)); + const seenTargets = new Set(); + let terminated = 0; + for (const candidate of candidates) { + const targetKey = candidate.processGroupId ? `pgid:${candidate.processGroupId}` : `pid:${candidate.pid}`; + if (seenTargets.has(targetKey)) continue; + seenTargets.add(targetKey); + await terminate({ + pid: candidate.pid, + processGroupId: candidate.processGroupId, + }); + terminated += 1; + } + + return { + scanned: processSnapshots.length, + matched: candidates.length, + terminated, + dryRun: false, + candidates, + }; +} + export async function findLocalServiceRegistryRecordByRuntimeServiceId(input: { runtimeServiceId: string; profileKind?: string; diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts index 866e1898317..ba27906ad6b 100644 --- a/server/src/services/workspace-runtime.ts +++ b/server/src/services/workspace-runtime.ts @@ -17,6 +17,7 @@ import { and, desc, eq, inArray } from "drizzle-orm"; import { asNumber, asString, parseObject, renderTemplate } from "../adapters/utils.js"; import { resolveHomeAwarePath } from "../home-paths.js"; import { + cleanupOrphanedPaperclipRuntimeProcesses, createLocalServiceKey, findLocalServiceRegistryRecordByRuntimeServiceId, findAdoptableLocalService, @@ -26,6 +27,7 @@ import { touchLocalServiceRegistryRecord, writeLocalServiceRegistryRecord, } from "./local-service-supervisor.js"; +import type { OrphanedPaperclipRuntimeProcess } from "./local-service-supervisor.js"; import type { WorkspaceOperationRecorder } from "./workspace-operations.js"; import { readExecutionWorkspaceConfig } from "./execution-workspaces.js"; import { readProjectWorkspaceRuntimeConfig } from "./project-workspace-runtime-config.js"; @@ -2774,6 +2776,12 @@ async function startLocalRuntimeService(input: { reuseKey: input.reuseKey, }, }); + const runtimeServiceId = stoppedReuseCandidate?.id ?? randomUUID(); + env.PAPERCLIP_MANAGED_RUNTIME = "workspace-runtime"; + env.PAPERCLIP_LOCAL_SERVICE_KEY = serviceKey; + env.PAPERCLIP_RUNTIME_SERVICE_ID = runtimeServiceId; + env.PAPERCLIP_RUNTIME_SERVICE_NAME = serviceName; + const adoptedRecord = await findAdoptableLocalService({ serviceKey, profileKind: "workspace-runtime", @@ -2879,7 +2887,7 @@ async function startLocalRuntimeService(input: { } const record: RuntimeServiceRecord = { - id: stoppedReuseCandidate?.id ?? randomUUID(), + id: runtimeServiceId, companyId: input.agent.companyId, projectId: input.workspace.projectId, projectWorkspaceId: input.workspace.workspaceId, @@ -3428,6 +3436,38 @@ export async function listWorkspaceRuntimeServicesForProjectWorkspaces( return grouped; } +async function listKnownWorkspaceRuntimeRoots(db: Db) { + const roots = new Set(); + const projectRows = await db + .select({ cwd: projectWorkspaces.cwd }) + .from(projectWorkspaces); + for (const row of projectRows) { + if (row.cwd) roots.add(row.cwd); + } + + const executionRows = await db + .select({ cwd: executionWorkspaces.cwd }) + .from(executionWorkspaces); + for (const row of executionRows) { + if (row.cwd) roots.add(row.cwd); + } + + return Array.from(roots); +} + +function listInMemoryRuntimeServiceProcessIds() { + const ids: number[] = []; + for (const record of runtimeServicesById.values()) { + if (record.child?.pid) ids.push(record.child.pid); + if (record.providerRef) { + const providerPid = Number.parseInt(record.providerRef, 10); + if (Number.isInteger(providerPid) && providerPid > 0) ids.push(providerPid); + } + if (record.processGroupId) ids.push(record.processGroupId); + } + return ids; +} + export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) { const rows = await db .select() @@ -3439,8 +3479,6 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) { ), ); - if (rows.length === 0) return { reconciled: 0, adopted: 0, stopped: 0 }; - let adopted = 0; let stopped = 0; for (const row of rows) { @@ -3546,7 +3584,22 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) { stopped += 1; } - return { reconciled: rows.length, adopted, stopped }; + const orphanedProcesses = + process.env.PAPERCLIP_STALE_RUNTIME_CLEANUP === "false" + ? { matched: 0, terminated: 0, candidates: [] as OrphanedPaperclipRuntimeProcess[] } + : await cleanupOrphanedPaperclipRuntimeProcesses({ + containmentRoots: await listKnownWorkspaceRuntimeRoots(db), + additionalManagedProcessIds: listInMemoryRuntimeServiceProcessIds(), + }); + + return { + reconciled: rows.length, + adopted, + stopped, + orphanedProcessesMatched: orphanedProcesses.matched, + orphanedProcessesCleaned: orphanedProcesses.terminated, + orphanedProcessCandidates: orphanedProcesses.candidates, + }; } export async function restartDesiredRuntimeServicesOnStartup(db: Db) {