diff --git a/server/scripts/benchmark-health-server-info.ts b/server/scripts/benchmark-health-server-info.ts new file mode 100644 index 00000000000..c447ee7dd58 --- /dev/null +++ b/server/scripts/benchmark-health-server-info.ts @@ -0,0 +1,211 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import type { AddressInfo } from "node:net"; +import os from "node:os"; +import path from "node:path"; +import express from "express"; +import type { Db } from "@paperclipai/db"; + +const argument = (name: string) => process.argv.find((value) => value.startsWith(`${name}=`))?.slice(name.length + 1); +const requests = Number(argument("--requests") ?? 200); +const pollIntervalMs = Number(argument("--poll-interval-ms") ?? 5000); +assert.ok(Number.isInteger(requests) && requests >= 2 && requests <= 100_000, "--requests must be an integer from 2 to 100000"); +assert.ok(Number.isInteger(pollIntervalMs) && pollIntervalMs >= 1, "--poll-interval-ms must be a positive integer"); +assert.notEqual(process.platform, "win32", "the counting git stand-in needs a POSIX shell"); + +const BUILD_COMMIT = "0123456789abcdef0123456789abcdef01234567"; +const COMMIT_SUBJECT = "Benchmark checkout commit"; +const COMMIT_DATE = "2026-11-12T00:00:00Z"; + +const realGit = execFileSync("/bin/sh", ["-c", "command -v git"], { encoding: "utf8" }).trim(); +const home = realpathSync(mkdtempSync(path.join(os.tmpdir(), "paperclip-health-server-info-benchmark-"))); +const execLogPath = path.join(home, "git-execs.log"); +const binDir = path.join(home, "bin"); +const packagedDir = path.join(home, "packaged-install"); +const checkoutDir = path.join(home, "source-checkout"); + +const gitExecs = () => existsSync(execLogPath) ? readFileSync(execLogPath, "utf8").split("\n").filter(Boolean).length : 0; +const sha256 = (value: string) => createHash("sha256").update(value).digest("hex").slice(0, 12); + +function isolateEnvironment() { + for (const name of Object.keys(process.env)) { + if (name.startsWith("PAPERCLIP_") || name.startsWith("GIT_")) delete process.env[name]; + } + process.env.PAPERCLIP_HOME = home; + process.env.PAPERCLIP_INSTANCE_ID = "default"; + process.env.PAPERCLIP_LOG_LEVEL = "silent"; + process.env.PATH = `${binDir}${path.delimiter}${process.env.PATH ?? ""}`; +} + +function createCheckout(): string { + mkdirSync(checkoutDir); + const git = (...args: string[]) => execFileSync(realGit, args, { + cwd: checkoutDir, + encoding: "utf8", + env: { + ...process.env, + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_CONFIG_SYSTEM: "/dev/null", + GIT_AUTHOR_NAME: "Benchmark", + GIT_AUTHOR_EMAIL: "benchmark@example.com", + GIT_AUTHOR_DATE: COMMIT_DATE, + GIT_COMMITTER_NAME: "Benchmark", + GIT_COMMITTER_EMAIL: "benchmark@example.com", + GIT_COMMITTER_DATE: COMMIT_DATE, + }, + }).trim(); + git("init", "--quiet", "--initial-branch=benchmark"); + writeFileSync(path.join(checkoutDir, "README.md"), "benchmark\n"); + git("add", "README.md"); + git("commit", "--quiet", "--message", COMMIT_SUBJECT); + return git("rev-parse", "HEAD"); +} + +function stubDatabase(): Db { + const rows = (value: unknown[]) => Object.assign(Promise.resolve(value), { groupBy: () => Promise.resolve([]) }); + return { + execute: async () => [{ "?column?": 1 }], + select: () => ({ from: () => ({ where: () => rows([{ count: 1 }]) }) }), + } as unknown as Db; +} + +try { + isolateEnvironment(); + mkdirSync(binDir); + mkdirSync(packagedDir); + const checkoutCommit = createCheckout(); + const standIn = path.join(binDir, "git"); + writeFileSync(standIn, `#!/bin/sh\necho exec >> '${execLogPath}'\nexec '${realGit}' "$@"\n`); + chmodSync(standIn, 0o755); + + const { healthRoutes } = await import("../src/routes/health.js"); + const { livenessRoutes } = await import("../src/routes/liveness.js"); + const { resetServerInfoCacheForTests } = await import("../src/server-info.js"); + const { setStartupRecoveryPhase } = await import("../src/startup-recovery-state.js"); + isolateEnvironment(); + setStartupRecoveryPhase("ready"); + + const realNow = Date.now; + let clockOffsetMs = 0; + Date.now = () => realNow() + clockOffsetMs; + + const scenarios = [ + { + name: "packaged install: no checkout, build supplies the commit", + directory: packagedDir, + buildCommit: BUILD_COMMIT, + expectedGit: { + available: true, + fullSha: BUILD_COMMIT, + shortSha: BUILD_COMMIT.slice(0, 7), + branchName: null, + subject: "Source build", + committedAt: null, + localChanges: { available: false, unavailableReason: "git_status_unavailable" }, + }, + }, + { + name: "source checkout", + directory: checkoutDir, + buildCommit: undefined, + expectedGit: { + available: true, + fullSha: checkoutCommit, + shortSha: checkoutCommit.slice(0, 7), + branchName: "benchmark", + subject: COMMIT_SUBJECT, + committedAt: new Date(COMMIT_DATE).toISOString(), + localChanges: { available: true, hasLocalChanges: false, stagedFileCount: 0, unstagedFileCount: 0, untrackedFileCount: 0 }, + }, + }, + ]; + const patterns = [ + { name: "burst", advanceMs: 0 }, + { name: `polled every ${pollIntervalMs} ms`, advanceMs: pollIntervalMs }, + ]; + const rows: Array> = []; + + for (const scenario of scenarios) { + process.chdir(scenario.directory); + if (scenario.buildCommit) process.env.PAPERCLIP_BUILD_COMMIT = scenario.buildCommit; + else delete process.env.PAPERCLIP_BUILD_COMMIT; + + for (const pattern of patterns) { + resetServerInfoCacheForTests(); + const execsBeforeRoutes = gitExecs(); + const app = express(); + app.use((req, _res, next) => { + if (req.get("x-benchmark-actor") === "board") { + (req as unknown as { actor: { type: string } }).actor = { type: "board" }; + } + next(); + }); + app.use("/api/liveness", livenessRoutes()); + app.use("/api/health", healthRoutes(stubDatabase(), { + deploymentMode: "authenticated", + deploymentExposure: "private", + authReady: true, + companyDeletionEnabled: false, + isWarmStandby: () => false, + })); + const server = app.listen(0, "127.0.0.1"); + await new Promise((resolve) => server.once("listening", resolve)); + const origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + const routeExecs = gitExecs() - execsBeforeRoutes; + + const bodies = { anonymous: new Set(), board: new Set(), liveness: new Set() }; + const execsBeforeRequests = gitExecs(); + const started = process.hrtime.bigint(); + for (let index = 0; index < requests; index += 1) { + clockOffsetMs += pattern.advanceMs; + const board = index % 2 === 1; + const response = await fetch(`${origin}/api/health`, board ? { headers: { "x-benchmark-actor": "board" } } : undefined); + const text = await response.text(); + assert.equal(response.status, 200, `${scenario.name}: health status`); + const body = JSON.parse(text); + assert.equal(body.commit, scenario.expectedGit.fullSha, `${scenario.name}: commit`); + if (board) { + assert.deepEqual(body.serverInfo.git, scenario.expectedGit, `${scenario.name}: serverInfo.git`); + assert.equal(new Date(body.serverInfo.processStartedAt).toISOString(), body.serverInfo.processStartedAt); + bodies.board.add(JSON.stringify({ commit: body.commit, serverInfo: body.serverInfo })); + } else { + assert.equal(body.serverInfo, undefined, `${scenario.name}: anonymous response carries no serverInfo`); + bodies.anonymous.add(text); + } + } + const totalMs = Number(process.hrtime.bigint() - started) / 1e6; + const requestExecs = gitExecs() - execsBeforeRequests; + + const liveness = await fetch(`${origin}/api/liveness`); + const livenessText = await liveness.text(); + assert.equal(liveness.status, 200, `${scenario.name}: liveness status`); + assert.deepEqual(JSON.parse(livenessText), { status: "ok", commit: scenario.expectedGit.fullSha }); + bodies.liveness.add(livenessText); + await new Promise((resolve) => { server.close(() => resolve()); server.closeAllConnections(); }); + + assert.equal(bodies.anonymous.size, 1, `${scenario.name}: anonymous health body is stable`); + assert.equal(bodies.board.size, 1, `${scenario.name}: board health serverInfo is stable`); + rows.push({ + scenario: scenario.name, + pattern: pattern.name, + "health requests": requests, + "git execs at route creation": routeExecs, + "git execs for requests": requestExecs, + "ms per request": Number((totalMs / requests).toFixed(3)), + "body sha256 (anonymous / board commit+serverInfo.git / liveness)": [ + sha256([...bodies.anonymous][0]!), + sha256(JSON.stringify(JSON.parse([...bodies.board][0]!).serverInfo.git)), + sha256(livenessText), + ].join(" / "), + }); + } + } + Date.now = realNow; + console.table(rows); + console.log(`node ${process.version}, ${os.platform()} ${os.arch()}, rss ${Math.round(process.memoryUsage().rss / 1e6)} MB`); +} finally { + process.chdir(os.tmpdir()); + rmSync(home, { recursive: true, force: true }); +} diff --git a/server/src/__tests__/health-server-info-git-probe.test.ts b/server/src/__tests__/health-server-info-git-probe.test.ts new file mode 100644 index 00000000000..d397fc76af4 --- /dev/null +++ b/server/src/__tests__/health-server-info-git-probe.test.ts @@ -0,0 +1,219 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import express from "express"; +import request from "supertest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { Db } from "@paperclipai/db"; +import type { ServerGitInfo } from "@paperclipai/shared"; + +const BUILD_COMMIT = "0123456789abcdef0123456789abcdef01234567"; +const POLL_INTERVAL_MS = 5000; + +const serverGit = vi.hoisted(() => ({ execFileSync: vi.fn() })); + +vi.mock("node:child_process", async (importOriginal) => { + const actual = await importOriginal(); + serverGit.execFileSync.mockImplementation((...args: unknown[]) => + (actual.execFileSync as (...values: unknown[]) => unknown)(...args)); + return { ...actual, execFileSync: serverGit.execFileSync }; +}); + +const { healthRoutes } = await import("../routes/health.js"); +const { livenessRoutes } = await import("../routes/liveness.js"); +const { getServerProcessStartedAt, resetServerInfoCacheForTests } = await import("../server-info.js"); +const { resetStartupRecoveryStateForTests, setStartupRecoveryPhase } = await import("../startup-recovery-state.js"); + +type HealthOptions = NonNullable[1]>; + +const healthOptionsAsAppPassesThem: HealthOptions = { + deploymentMode: "authenticated", + deploymentExposure: "private", + authReady: true, + companyDeletionEnabled: false, + isWarmStandby: () => false, +}; + +function serverGitStarts(): number { + return serverGit.execFileSync.mock.calls.filter(([command]) => command === "git").length; +} + +function fixtureGit(cwd: string, ...args: string[]) { + const result = spawnSync("git", args, { + cwd, + encoding: "utf8", + env: { + ...process.env, + GIT_CONFIG_GLOBAL: os.devNull, + GIT_CONFIG_SYSTEM: os.devNull, + GIT_AUTHOR_NAME: "Test", + GIT_AUTHOR_EMAIL: "test@example.com", + GIT_COMMITTER_NAME: "Test", + GIT_COMMITTER_EMAIL: "test@example.com", + }, + }); + return { status: result.status, stdout: result.stdout.trim() }; +} + +function commit(checkout: string, subject: string): string { + fs.writeFileSync(path.join(checkout, "file.txt"), `${subject}\n`); + expect(fixtureGit(checkout, "add", "file.txt").status).toBe(0); + expect(fixtureGit(checkout, "commit", "--quiet", "--message", subject).status).toBe(0); + return fixtureGit(checkout, "rev-parse", "HEAD").stdout; +} + +function createCheckout(): string { + const checkout = path.join(home, "checkout"); + fs.mkdirSync(path.join(checkout, "server"), { recursive: true }); + expect(fixtureGit(checkout, "init", "--quiet", "--initial-branch=main").status).toBe(0); + return checkout; +} + +function stubDatabase(): Db { + const rows = (value: unknown[]) => Object.assign(Promise.resolve(value), { groupBy: () => Promise.resolve([]) }); + return { + execute: vi.fn().mockResolvedValue([{ "?column?": 1 }]), + select: () => ({ from: () => ({ where: () => rows([{ count: 1 }]) }) }), + } as unknown as Db; +} + +function createServer(serverInfo?: HealthOptions["serverInfo"]) { + const app = express(); + app.use((req, _res, next) => { + if (req.get("x-test-actor") === "board") { + (req as unknown as { actor: { type: string } }).actor = { type: "board" }; + } + next(); + }); + app.use("/api/liveness", serverInfo ? livenessRoutes(serverInfo) : livenessRoutes()); + app.use("/api/health", healthRoutes(stubDatabase(), { ...healthOptionsAsAppPassesThem, ...(serverInfo ? { serverInfo } : {}) })); + return app; +} + +async function pollAfterCacheExpiry(app: express.Express, polls = 1) { + const responses: Array<{ anonymous: string; board: string; liveness: string }> = []; + for (let index = 0; index < polls; index += 1) { + vi.setSystemTime(Date.now() + POLL_INTERVAL_MS); + const anonymous = await request(app).get("/api/health").expect(200); + const board = await request(app).get("/api/health").set("x-test-actor", "board").expect(200); + const liveness = await request(app).get("/api/liveness").expect(200); + responses.push({ anonymous: anonymous.text, board: board.text, liveness: liveness.text }); + } + return responses; +} + +const originalDirectory = process.cwd(); +let home: string; + +beforeEach(() => { + home = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-git-probe-"))); + for (const name of Object.keys(process.env)) { + if (name.startsWith("GIT_") || name.startsWith("PAPERCLIP_CLOUD_")) vi.stubEnv(name, undefined); + } + vi.stubEnv("PAPERCLIP_BUILD_COMMIT", undefined); + vi.useFakeTimers({ toFake: ["Date"] }); + serverGit.execFileSync.mockClear(); + resetServerInfoCacheForTests(); + setStartupRecoveryPhase("ready"); +}); + +afterEach(() => { + process.chdir(originalDirectory); + vi.useRealTimers(); + vi.unstubAllEnvs(); + resetServerInfoCacheForTests(); + resetStartupRecoveryStateForTests(); + fs.rmSync(home, { recursive: true, force: true }); +}); + +describe("git processes behind health and liveness requests", () => { + it("starts no git process when the server does not run from a checkout", async () => { + const install = path.join(home, "app"); + fs.mkdirSync(install); + process.chdir(install); + vi.stubEnv("PAPERCLIP_BUILD_COMMIT", BUILD_COMMIT); + const realGitShowOutsideCheckout = fixtureGit(install, "show", "-s", "--format=%H", "HEAD"); + expect(realGitShowOutsideCheckout.status).not.toBe(0); + const buildGitInfo: ServerGitInfo = { + available: true, + fullSha: BUILD_COMMIT, + shortSha: BUILD_COMMIT.slice(0, 7), + branchName: null, + subject: "Source build", + committedAt: null, + localChanges: { available: false, unavailableReason: "git_status_unavailable" }, + }; + + const responses = await pollAfterCacheExpiry(createServer(), 5); + + const serverWithInjectedSnapshot = createServer({ processStartedAt: getServerProcessStartedAt(), git: buildGitInfo }); + const [reference] = await pollAfterCacheExpiry(serverWithInjectedSnapshot); + for (const response of responses) expect(response).toEqual(reference); + expect(JSON.parse(reference!.anonymous)).toMatchObject({ status: "ok", commit: BUILD_COMMIT }); + expect(JSON.parse(reference!.anonymous).serverInfo).toBeUndefined(); + expect(JSON.parse(reference!.board).serverInfo).toEqual({ + processStartedAt: getServerProcessStartedAt(), + git: buildGitInfo, + }); + expect(JSON.parse(reference!.liveness)).toEqual({ status: "ok", commit: BUILD_COMMIT }); + expect(serverGitStarts()).toBe(0); + }); + + it("reads a checkout again after its commit changes, also from a subdirectory", async () => { + const checkout = createCheckout(); + const firstCommit = commit(checkout, "First commit"); + process.chdir(path.join(checkout, "server")); + const app = createServer(); + + const [first] = await pollAfterCacheExpiry(app); + expect(JSON.parse(first!.board).serverInfo.git).toMatchObject({ + fullSha: firstCommit, + branchName: "main", + subject: "First commit", + localChanges: { available: true, hasLocalChanges: false }, + }); + + const secondCommit = commit(checkout, "Second commit"); + const [second] = await pollAfterCacheExpiry(app); + expect(JSON.parse(second!.anonymous).commit).toBe(secondCommit); + expect(JSON.parse(second!.board).serverInfo).toMatchObject({ + processStartedAt: getServerProcessStartedAt(), + git: { fullSha: secondCommit, subject: "Second commit" }, + }); + const commitReadAtRouteCreation = firstCommit; + expect(JSON.parse(second!.liveness)).toEqual({ status: "ok", commit: commitReadAtRouteCreation }); + }); + + it("reads a linked worktree, where .git is a file", async () => { + const checkout = createCheckout(); + commit(checkout, "First commit"); + const worktree = path.join(home, "worktree"); + expect(fixtureGit(checkout, "worktree", "add", "--quiet", "-b", "feature", worktree).status).toBe(0); + const worktreeCommit = commit(worktree, "Worktree commit"); + expect(fs.statSync(path.join(worktree, ".git")).isFile()).toBe(true); + process.chdir(worktree); + + const [response] = await pollAfterCacheExpiry(createServer()); + + expect(JSON.parse(response!.board).serverInfo.git).toMatchObject({ + fullSha: worktreeCommit, + branchName: "feature", + subject: "Worktree commit", + }); + }); + + it("still asks git when GIT_DIR names a repository outside the working directory", async () => { + const checkout = createCheckout(); + const headCommit = commit(checkout, "Commit found through GIT_DIR"); + const install = path.join(home, "app"); + fs.mkdirSync(install); + process.chdir(install); + vi.stubEnv("GIT_DIR", path.join(checkout, ".git")); + + const [response] = await pollAfterCacheExpiry(createServer()); + + expect(JSON.parse(response!.anonymous).commit).toBe(headCommit); + expect(serverGitStarts()).toBeGreaterThan(0); + }); +}); diff --git a/server/src/server-info.ts b/server/src/server-info.ts index 6d3e861e6bb..845df68e7b8 100644 --- a/server/src/server-info.ts +++ b/server/src/server-info.ts @@ -1,4 +1,6 @@ import { execFileSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { dirname, join } from "node:path"; import type { ServerGitInfo, ServerGitLocalChanges, ServerInfoSnapshot } from "@paperclipai/shared"; import { parseBuildCommit, readBuildCommit } from "./build-commit.js"; @@ -9,7 +11,21 @@ type BuildCommitCommand = () => string | null; const SHORT_SHA_RE = /^[0-9a-f]{7,40}$/i; +function workingDirectoryIsInsideGitCheckout(): boolean { + if (process.env.GIT_DIR) return true; + let directory = process.cwd(); + for (;;) { + if (existsSync(join(directory, ".git"))) return true; + const parent = dirname(directory); + if (parent === directory) return false; + directory = parent; + } +} + function defaultGitCommand() { + if (!workingDirectoryIsInsideGitCheckout()) { + throw new Error("The working directory is not inside a Git checkout"); + } return execFileSync( "git", ["show", "-s", "--format=%H%n%h%n%s%n%cI", "HEAD"],