From cc96c9f83281ff5ea0fdabfb1873a3bcdb029640 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:46:50 -0500 Subject: [PATCH] fix(server): probe libc asynchronously before serving requests (#16335) Linux runner qualification generated a full Node diagnostic report to read one libc version field. Reverse DNS on open sockets could stop the event loop for minutes. Replace report generation with a one-time asynchronous getconf probe during module initialization. Bound it to one second and 1 KiB of output, cache the measured facts, and preserve unknown-libc failure behavior. Verification: 149 focused tests, repository typecheck and build, full CI, and fresh Greptile 5/5 with no comments. A timed-out local heartbeat fixture passed when rerun alone. Co-Authored-By: Paperclip --- doc/DEVELOPING.md | 12 ++ .../__tests__/agent-runner-platform.test.ts | 121 +++++++++++------- .../dev-runner-native-binary.test.ts | 6 +- server/src/lib/agent-runner-platform.ts | 63 ++++----- .../services/agent-runner-selection.test.ts | 17 ++- 5 files changed, 126 insertions(+), 93 deletions(-) diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index f647240ce5a..7b07159f3b3 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -679,6 +679,18 @@ Activity-only reorderings wait for one second without further activity changes; new and removed tasks appear immediately. Titles, status, and live indicators stay current during that delay. +### Local Linux runner qualification + +Before serving requests, the server reads `getconf GNU_LIBC_VERSION` once with +an asynchronous subprocess, a one-second timeout, and a 1 KiB output limit. +Runner discovery reuses those immutable platform facts. This check does not +generate a Node diagnostic report or resolve socket hostnames. + +If `getconf` is missing, fails, times out, or returns an unrecognized version, +the local libc remains unknown and automatic GNU/Linux runner selection stays +disabled. Qualified Linux targets still require GNU libc 2.39 or newer. Existing +saved execution choices remain unchanged. + ### OpenCode native runtime Paperclip Runner resolves the pinned OpenCode platform package installed with diff --git a/server/src/__tests__/agent-runner-platform.test.ts b/server/src/__tests__/agent-runner-platform.test.ts index c309d79ddcb..7657d098736 100644 --- a/server/src/__tests__/agent-runner-platform.test.ts +++ b/server/src/__tests__/agent-runner-platform.test.ts @@ -1,61 +1,92 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { localAgentRunnerTarget, resetLocalAgentRunnerTargetCacheForTests } from "../lib/agent-runner-platform.js"; -const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform")!; -const report = process.report as NodeJS.ProcessReport & { excludeNetwork: boolean }; +const probe = vi.hoisted(() => ({ execFile: vi.fn() })); +vi.mock("node:child_process", () => ({ execFile: probe.execFile })); +const platform = Object.getOwnPropertyDescriptor(process, "platform")!; +const architecture = Object.getOwnPropertyDescriptor(process, "arch")!; -describe("localAgentRunnerTarget", () => { - beforeEach(() => { - Object.defineProperty(process, "platform", { ...originalPlatform, value: "linux" }); - resetLocalAgentRunnerTargetCacheForTests(); - report.excludeNetwork = false; +beforeEach(() => { + vi.resetModules(); + probe.execFile.mockReset(); + Object.defineProperty(process, "platform", { ...platform, value: "linux" }); + Object.defineProperty(process, "arch", { ...architecture, value: "x64" }); + vi.spyOn(process.report, "getReport").mockImplementation(() => { + throw new Error("Diagnostic reports must not be used for platform detection"); }); +}); - afterEach(() => { - Object.defineProperty(process, "platform", originalPlatform); - resetLocalAgentRunnerTargetCacheForTests(); - vi.restoreAllMocks(); +afterEach(() => { + vi.restoreAllMocks(); + Object.defineProperty(process, "platform", platform); + Object.defineProperty(process, "arch", architecture); +}); + +function reply(stdout: string, error: Error | null = null) { + probe.execFile.mockImplementation((_command, _args, _options, callback) => { + callback(error, stdout, ""); }); +} - it("reads glibc from a report that excludes network data, then restores the setting", () => { - const excludeNetworkDuringReport: boolean[] = []; - vi.spyOn(process.report, "getReport").mockImplementation(() => { - excludeNetworkDuringReport.push(report.excludeNetwork); - return { header: { glibcVersionRuntime: "2.36" } } as never; +describe("local runner platform probe", () => { + it.each([ + ["glibc 2.38\n", "2.38", false], + ["glibc 2.39\n", "2.39", true], + [" glibc 2.41\n", "2.41", true], + ["glibc 2.39.1", "2.39.1", true], + ] as const)("uses getconf's measured version %s", async (stdout, version, supported) => { + reply(stdout); + const module = await import("../lib/agent-runner-platform.js"); + expect(module.localAgentRunnerTarget()).toEqual({ + driver: "local", platform: "linux", architecture: "x64", + libc: "glibc", glibcVersion: version, }); - - expect(localAgentRunnerTarget()).toMatchObject({ platform: "linux", libc: "glibc", glibcVersion: "2.36" }); - // Network data makes the report reverse-resolve every open socket with - // synchronous DNS, which can stall the server. - expect(excludeNetworkDuringReport).toEqual([true]); - expect(report.excludeNetwork).toBe(false); + expect(module.localPaperclipRunnerSupported()).toBe(supported); + expect(process.report.getReport).not.toHaveBeenCalled(); }); - - it("generates the report once per process", () => { - const getReport = vi.spyOn(process.report, "getReport") - .mockReturnValue({ header: { glibcVersionRuntime: "2.36" } } as never); - - for (let index = 0; index < 5; index += 1) localAgentRunnerTarget(); - expect(getReport).toHaveBeenCalledTimes(1); + it("probes once before discovery, with a bounded asynchronous command", async () => { + let callback!: (error: Error | null, stdout: string, stderr: string) => void; + probe.execFile.mockImplementation((_command, _args, _options, done) => { callback = done; }); + const loaded = import("../lib/agent-runner-platform.js"); + await vi.waitFor(() => expect(probe.execFile).toHaveBeenCalledTimes(1)); + await new Promise(resolve => setTimeout(resolve, 0)); + callback(null, "glibc 2.41\n", ""); + const module = await loaded; + const first = module.localAgentRunnerTarget(); + first.glibcVersion = "0.0"; + for (let index = 0; index < 10; index++) { + expect(module.localAgentRunnerTarget().glibcVersion).toBe("2.41"); + } + expect(probe.execFile).toHaveBeenCalledTimes(1); + expect(probe.execFile).toHaveBeenCalledWith("getconf", ["GNU_LIBC_VERSION"], { + encoding: "utf8", timeout: 1_000, killSignal: "SIGKILL", maxBuffer: 1_024, + }, expect.any(Function)); }); - it("reports an unknown libc once when the report fails, and restores the setting", () => { - const getReport = vi.spyOn(process.report, "getReport").mockImplementation(() => { - throw new Error("report unavailable"); - }); - - expect(localAgentRunnerTarget()).toMatchObject({ libc: "unknown" }); - expect(localAgentRunnerTarget()).toMatchObject({ libc: "unknown" }); - expect(getReport).toHaveBeenCalledTimes(1); - expect(report.excludeNetwork).toBe(false); - }); + it.each(["", "musl 1.2.5", "glibc malformed", "glibc 2.39ubuntu1", "glibc 2.41\nprivate-data"])( + "fails closed for unrecognized output %s", async stdout => { + reply(stdout); + const module = await import("../lib/agent-runner-platform.js"); + expect(module.localAgentRunnerTarget()).toMatchObject({ libc: "unknown" }); + expect(module.localAgentRunnerTarget()).not.toHaveProperty("glibcVersion"); + expect(module.localPaperclipRunnerSupported()).toBe(false); + }, + ); - it("does not generate a report off Linux", () => { - Object.defineProperty(process, "platform", { ...originalPlatform, value: "darwin" }); - const getReport = vi.spyOn(process.report, "getReport"); + it.each(["ENOENT", "ETIMEDOUT", "ERR_CHILD_PROCESS_STDIO_MAXBUFFER"])( + "keeps discovery available and fails closed on %s", async code => { + reply("", Object.assign(new Error("probe failed"), { code })); + const module = await import("../lib/agent-runner-platform.js"); + expect(module.localAgentRunnerTarget()).toMatchObject({ libc: "unknown" }); + expect(module.localPaperclipRunnerSupported()).toBe(false); + expect(probe.execFile).toHaveBeenCalledTimes(1); + }, + ); - expect(localAgentRunnerTarget()).toEqual({ driver: "local", platform: "darwin", architecture: process.arch }); - expect(getReport).not.toHaveBeenCalled(); + it.each(["darwin", "win32"])("does not probe libc on %s", async value => { + Object.defineProperty(process, "platform", { ...platform, value }); + const module = await import("../lib/agent-runner-platform.js"); + expect(module.localAgentRunnerTarget()).toEqual({ driver: "local", platform: value, architecture: "x64" }); + expect(probe.execFile).not.toHaveBeenCalled(); }); }); diff --git a/server/src/__tests__/dev-runner-native-binary.test.ts b/server/src/__tests__/dev-runner-native-binary.test.ts index a460176f433..6b0e7695b19 100644 --- a/server/src/__tests__/dev-runner-native-binary.test.ts +++ b/server/src/__tests__/dev-runner-native-binary.test.ts @@ -2,7 +2,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { localPaperclipRunnerSupported, resetLocalAgentRunnerTargetCacheForTests } from "../lib/agent-runner-platform.js"; +import { paperclipRunnerSupportsPlatform } from "@paperclipai/shared"; import { paperclipRunnerBinaryNeedsBuild, @@ -16,7 +16,6 @@ const initialArchitecture = Object.getOwnPropertyDescriptor(process, "arch")!; afterEach(() => { vi.restoreAllMocks(); - resetLocalAgentRunnerTargetCacheForTests(); Object.defineProperty(process, "platform", initialPlatform); Object.defineProperty(process, "arch", initialArchitecture); for (const root of tempRoots) { @@ -66,8 +65,7 @@ describe("paperclip runner native dev prerequisite", () => { it.each(["x64", "arm64"].flatMap(architecture => [[undefined, false], ["2.38", false], ["2.39", true], ["3.0", true]].map(([glibcVersionRuntime, native]) => ({ architecture, glibcVersionRuntime, native }))))("uses the measured libc prerequisite on Linux $architecture/$glibcVersionRuntime when no native run or flag requires it", ({ architecture, glibcVersionRuntime, native }) => { Object.defineProperty(process, "platform", { ...initialPlatform, value: "linux" }); Object.defineProperty(process, "arch", { ...initialArchitecture, value: architecture }); - vi.spyOn(process.report, "getReport").mockReturnValue({ header: { glibcVersionRuntime } } as never); - const nativeRunnerRequired = localPaperclipRunnerSupported(); + const nativeRunnerRequired = paperclipRunnerSupportsPlatform("codex_local", "linux", architecture, glibcVersionRuntime ? "glibc" : "unknown", glibcVersionRuntime as string | undefined); expect(nativeRunnerRequired).toBe(native); expect(paperclipRunnerBinaryNeedsBuild({ repoRoot: "/checkout/without/a/staged/binary", nativeRunnerRequired })).toBe(native); }); diff --git a/server/src/lib/agent-runner-platform.ts b/server/src/lib/agent-runner-platform.ts index 3a48fef06d5..efc9eeb4426 100644 --- a/server/src/lib/agent-runner-platform.ts +++ b/server/src/lib/agent-runner-platform.ts @@ -1,46 +1,39 @@ +import { execFile } from "node:child_process"; import { paperclipRunnerSupportsPlatform } from "@paperclipai/shared"; -type LinuxLibc = { libc: "glibc" | "unknown"; glibcVersion?: string }; +type LocalRunnerTarget = { + driver: "local"; + platform: NodeJS.Platform; + architecture: string; + libc?: "glibc" | "unknown"; + glibcVersion?: string; +}; -// `excludeNetwork` is part of the Node 24 report API, but the installed Node -// type definitions do not declare it yet. -type ProcessReportWithNetworkSetting = NodeJS.ProcessReport & { excludeNetwork: boolean }; - -let cachedLinuxLibc: LinuxLibc | undefined; - -// The C library cannot change while the process runs, so detect it once. -// `process.report.getReport()` blocks the event loop. Unless `excludeNetwork` -// is set, it also resolves the endpoints of every open socket with synchronous -// reverse DNS. A slow resolver then stalls the server for seconds per socket, -// on every caller, so the report always excludes network data. -function detectLinuxLibc(): LinuxLibc { - if (cachedLinuxLibc) return cachedLinuxLibc; - let detected: LinuxLibc = { libc: "unknown" }; - const report = process.report as ProcessReportWithNetworkSetting; - const previousExcludeNetwork = report.excludeNetwork; +async function detectLocalRunnerTarget(): Promise { + const target: LocalRunnerTarget = { + driver: "local", platform: process.platform, architecture: process.arch, + }; + if (target.platform !== "linux") return target; + target.libc = "unknown"; try { - report.excludeNetwork = true; - const { header } = report.getReport() as { header?: { glibcVersionRuntime?: unknown } }; - if (typeof header?.glibcVersionRuntime === "string" && header.glibcVersionRuntime.trim()) { - detected = { libc: "glibc", glibcVersion: header.glibcVersionRuntime.trim() }; + const stdout = await new Promise((resolve, reject) => { + execFile("getconf", ["GNU_LIBC_VERSION"], { + encoding: "utf8", timeout: 1_000, killSignal: "SIGKILL", maxBuffer: 1_024, + }, (error, output) => error ? reject(error) : resolve(output)); + }); + const version = /^glibc\s+(\d+\.\d+(?:\.\d+)?)$/.exec(stdout.trim())?.[1]; + if (version) { + target.libc = "glibc"; + target.glibcVersion = version; } - } catch { - /* An unidentified libc must not select a GNU/Linux binary. */ - } finally { - report.excludeNetwork = previousExcludeNetwork; - } - cachedLinuxLibc = detected; - return detected; + } catch {} + return target; } -export function resetLocalAgentRunnerTargetCacheForTests(): void { - cachedLinuxLibc = undefined; -} +const localTarget = await detectLocalRunnerTarget(); -export function localAgentRunnerTarget(): { driver: "local"; platform: NodeJS.Platform; architecture: string; libc?: "glibc" | "unknown"; glibcVersion?: string } { - if (process.platform !== "linux") return { driver: "local", platform: process.platform, architecture: process.arch }; - const { libc, glibcVersion } = detectLinuxLibc(); - return { driver: "local", platform: process.platform, architecture: process.arch, libc, ...(glibcVersion === undefined ? {} : { glibcVersion }) }; +export function localAgentRunnerTarget(): LocalRunnerTarget { + return { ...localTarget }; } export function localPaperclipRunnerSupported(): boolean { diff --git a/server/src/services/agent-runner-selection.test.ts b/server/src/services/agent-runner-selection.test.ts index d893cada791..4c55b770f0c 100644 --- a/server/src/services/agent-runner-selection.test.ts +++ b/server/src/services/agent-runner-selection.test.ts @@ -11,12 +11,12 @@ vi.mock("@paperclipai/adapter-utils/ssh", () => ({ runSshCommand: state.ssh })); import { AGENT_RUNNER_SSH_PLATFORM_COMMAND, agentRunnerAvailability, resolveAgentRunnerTargetForCompany, resolveNewAgentRunner, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js"; import { buildCodexAcpConfig, resolveCodexExecutionEngine } from "@paperclipai/adapter-codex-local/server"; import { secretService } from "./secrets.js"; -import { localAgentRunnerTarget, resetLocalAgentRunnerTargetCacheForTests } from "../lib/agent-runner-platform.js"; +import * as localPlatform from "../lib/agent-runner-platform.js"; const target = { driver: "local", platform: "linux", architecture: "x64" }; const qualifiedTargets = [target, { driver: "local", platform: "linux", architecture: "arm64" }, { driver: "local", platform: "darwin", architecture: "arm64" }, { driver: "local", platform: "darwin", architecture: "x64" }]; const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform")!; const originalArchitecture = Object.getOwnPropertyDescriptor(process, "arch")!; -afterEach(() => { vi.restoreAllMocks(); resetLocalAgentRunnerTargetCacheForTests(); Object.defineProperty(process, "platform", originalPlatform); Object.defineProperty(process, "arch", originalArchitecture); }); +afterEach(() => { vi.restoreAllMocks(); Object.defineProperty(process, "platform", originalPlatform); Object.defineProperty(process, "arch", originalArchitecture); }); describe("server-owned Codex runner selection", () => { beforeEach(() => { state.disabled = []; state.overridden.clear(); state.settings = {}; state.getEnvironment.mockReset(); state.bindings.mockReset().mockResolvedValue([]); state.managedEnvironment.mockReset(); state.kubernetesEnvironment.mockReset().mockResolvedValue(null); state.ensureKubernetesEnvironment.mockReset(); state.ssh.mockReset().mockResolvedValue({ stdout: "Linux\nx86_64\n" }); state.sandboxQualification.mockReset().mockResolvedValue(null); state.resolveEnvironment.mockReset().mockResolvedValue({ driver: "ssh", config: {} }); }); @@ -33,13 +33,12 @@ describe("server-owned Codex runner selection", () => { expect(agentRunnerAvailability("codex_local", { ...target, libc: "glibc" }).defaultRunner).toBe("paperclip"); } }); - it.each([[undefined, false], ["2.38", false], ["2.39", true], ["3.0", true], [" 2.39 ", true], ["malformed", false], ["", false]] as const)("uses the measured local Node libc version (%s) for creation", async (glibcVersionRuntime, native) => { + it.each([[undefined, false], ["2.38", false], ["2.39", true], ["3.0", true], [" 2.39 ", true], ["malformed", false], ["", false]] as const)("uses the measured local libc version (%s) for creation", async (glibcVersionRuntime, native) => { Object.defineProperty(process, "platform", { ...originalPlatform, value: "linux" }); Object.defineProperty(process, "arch", { ...originalArchitecture, value: "arm64" }); - vi.spyOn(process.report, "getReport").mockReturnValue({ header: { glibcVersionRuntime }, environmentVariables: { PRIVATE_SENTINEL: "do-not-retain" } } as never); - const localTarget = localAgentRunnerTarget(); + vi.spyOn(localPlatform, "localAgentRunnerTarget").mockReturnValue({ driver: "local", platform: "linux", architecture: "arm64", libc: glibcVersionRuntime ? "glibc" : "unknown", ...(glibcVersionRuntime ? { glibcVersion: glibcVersionRuntime.trim() } : {}) }); + const localTarget = localPlatform.localAgentRunnerTarget(); expect(localTarget).toEqual({ driver: "local", platform: "linux", architecture: "arm64", libc: glibcVersionRuntime ? "glibc" : "unknown", ...(glibcVersionRuntime ? { glibcVersion: glibcVersionRuntime.trim() } : {}) }); - expect(JSON.stringify(localTarget)).not.toContain("do-not-retain"); expect(agentRunnerAvailability("codex_local").defaultRunner).toBe(native ? "paperclip" : "legacy"); const saved = await resolveNewAgentRunnerForCompany({} as never, "company", { adapterType: "codex_local" }); expect(saved.adapterType).toBe(native ? "paperclip_runner" : "codex_local"); @@ -61,10 +60,10 @@ describe("server-owned Codex runner selection", () => { expect(resolveNewAgentRunner(saved)).toEqual({ adapterType: saved.adapterType, adapterConfig: saved.adapterConfig }); } }); - it("keeps an unidentified local Linux libc legacy if Node's report is unavailable", () => { + it("keeps an unidentified local Linux libc legacy if the platform probe is unavailable", () => { Object.defineProperty(process, "platform", { ...originalPlatform, value: "linux" }); - vi.spyOn(process.report, "getReport").mockImplementation(() => { throw new Error("private-report-details"); }); - expect(localAgentRunnerTarget()).toMatchObject({ libc: "unknown" }); + vi.spyOn(localPlatform, "localAgentRunnerTarget").mockReturnValue({ driver: "local", platform: "linux", architecture: "x64", libc: "unknown" }); + expect(localPlatform.localAgentRunnerTarget()).toMatchObject({ libc: "unknown" }); expect(agentRunnerAvailability("codex_local").defaultRunner).toBe("legacy"); }); it.each(qualifiedTargets)("uses native Codex automatically with compatible configuration and is idempotent on $platform/$architecture", target => {