diff --git a/doc/plans/2026-11-13-thousand-concurrent-agents.md b/doc/plans/2026-11-13-thousand-concurrent-agents.md index 852a9fdf72a..825f67dcdfa 100644 --- a/doc/plans/2026-11-13-thousand-concurrent-agents.md +++ b/doc/plans/2026-11-13-thousand-concurrent-agents.md @@ -445,12 +445,44 @@ A missing source is reported as "not measured", never as zero. | Lease renewals that miss their deadline | finish line | 0 | metrics endpoint | | Kill one server process | finish line | runs resume, no Retry | not measured by this harness | -The server has no metrics endpoint today. `PAPERCLIP_SCALE_METRICS_URL` must -answer a JSON object with any of `mainThreadCpu`, `eventLoopUtilization` -(0 to 1, highest server process), `eventLoopDelayP99Ms`, -`statementsPerSecond`, `poolWaitP99Ms`, `runEventLatenessP99Ms`, -`agentApiPreHandlerP99Ms` and `leaseRenewalsMissed` (a counter). Without it, -the four server criteria are not measured, and a run can be at best +**Server metrics.** Each server process can serve +`GET /api/instance/server-metrics`, a JSON snapshot of that process over the +last 50 to 60 s (`windowMs`), with its `processId` (random per process +start), `uptimeSec` and `buildRevision`. It is off by default: set +`PAPERCLIP_SERVER_METRICS=true` on the stack before the run (read once at +start; off, the path answers 404 and nothing is collected). Only an instance +admin can read it, so the board token must belong to an instance admin. +Point the harness at it on the target origin, and the harness sends the board +token in its `Authorization: Bearer` header (never in the URL; +`PAPERCLIP_SCALE_METRICS_TOKEN` overrides it, and a URL on another origin gets +no token unless that variable is set): + +```sh +export PAPERCLIP_SCALE_METRICS_URL=https://$H/api/instance/server-metrics +``` + +| Field | Meaning | Source | +| --- | --- | --- | +| `mainThreadCpu` | main-thread CPU per wall second, 0 to 1 (Linux only, else null) | `/proc/self/task//stat` | +| `eventLoopUtilization` | 0 to 1 | `performance.eventLoopUtilization()` | +| `eventLoopDelayP99Ms` | highest 10 s p99 in the window | `monitorEventLoopDelay` (20 ms) | +| `statementsPerSecond` | Drizzle statements per second (not BEGIN/COMMIT or raw driver calls) | Drizzle logger counter | +| `runEventLatenessP99Ms` | runner `emittedAt` to committed row, p99 (null with no events) | runner event append | +| `agentApiPreHandlerP99Ms` | agent API request arrival to the API router, p99 (null with no agent requests) | two middlewares | +| `leaseRenewalsMissed` | counter since process start: native run leases whose deadline passed unrenewed | lease deadline timer | + +`processCpu`, `runEventLatenessSamples`, `agentApiPreHandlerSamples` and +`dedicatedConnections` are extra context. `dedicatedConnections` is the +process's dedicated-connection bound at the time of the request: `limit`, +`inUse`, `waiting` (gauges), `peakInUse`, `queued`, `overLimit` and +`nestedOverLimit` (counts since start); queueing there shows a restart burst. `poolWaitP99Ms` is absent: postgres.js exposes no cheap hook +for time spent queued for a connection, so pool wait stays not measured. With +several processes, each request reaches one of them; the harness keeps the +highest gauge and sums the counter's growth per `processId`; a process +whose `uptimeSec` shows it started during the run counts its whole counter. +After an event-loop block longer than the window, `windowMs` reports the real, +longer span so the block stays visible. Without the +endpoint, the four server criteria are not measured, and a run can be at best `incomplete`. The harness does not read the database. **Proof order.** A hosted stack that is not `paperclip.staging` first diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 9b84a1ce668..b8f1421ed29 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -65,3 +65,4 @@ export * from "./schema/index.js"; export { databaseWorkScopeKey, primeDatabaseWorkTransactionId, signalDatabaseWork, subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "./work-signals.js"; export { withDatabaseOperation } from "./response-capture.js"; +export { databaseStatementCount } from "./statement-count.js"; diff --git a/packages/db/src/lazy-transaction-queries.test.ts b/packages/db/src/lazy-transaction-queries.test.ts index 2cefdf6c8a6..4c7828fc796 100644 --- a/packages/db/src/lazy-transaction-queries.test.ts +++ b/packages/db/src/lazy-transaction-queries.test.ts @@ -5,6 +5,7 @@ import postgres from "postgres"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { createDb, type Db } from "./client.js"; import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "./response-capture.js"; +import { databaseStatementCount } from "./statement-count.js"; import { announcementDismissals, announcementPublications } from "./schema/index.js"; import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, @@ -48,6 +49,17 @@ describeEmbeddedPostgres("transactions on the Paperclip database client", () => }); afterAll(async () => { await database?.cleanup(); }); + it("counts each statement, inside transactions and savepoints too", async () => { + const before = databaseStatementCount(); + await db.select().from(announcementPublications); + await db.transaction(async (tx) => { + await publication(tx, "counted"); + await tx.transaction(async nested => { await nested.select().from(announcementPublications); }); + await tx.query.announcementPublications.findMany(); + }); + expect(databaseStatementCount() - before).toBe(4); + }); + describe("relational query builders", () => { it("builds none for a transaction or savepoint that never reads tx.query", async () => { const schemaTables = Object.keys(db.query).length; diff --git a/packages/db/src/lazy-transaction-queries.ts b/packages/db/src/lazy-transaction-queries.ts index e153ca6452b..c99a2deb19b 100644 --- a/packages/db/src/lazy-transaction-queries.ts +++ b/packages/db/src/lazy-transaction-queries.ts @@ -3,6 +3,7 @@ import { RelationalQueryBuilder } from "drizzle-orm/pg-core/query-builders/query import { drizzle, type PostgresJsDatabase } from "drizzle-orm/postgres-js"; import type { TableRelationalConfig } from "drizzle-orm"; import type postgres from "postgres"; +import { statementCountingLogger } from "./statement-count.js"; type Callback = (tx: Transaction) => unknown; type Transaction = { @@ -27,18 +28,19 @@ type Transaction = { * built, bound to that transaction's session, and `tx._` still describes the * schema. * - * Both clients and the builders' dialect take Drizzle's defaults. A Drizzle - * option added here (`casing`, `logger`, `cache`) must go to all three. + * Both clients count each statement through `statementCountingLogger`, and + * otherwise both clients and the builders' dialect take Drizzle's defaults. A + * Drizzle option added here (`casing`, `cache`) must go to all three. */ export function drizzleWithLazyTransactionQueries, TClient extends postgres.Sql>( client: TClient, schema: TSchema, ): PostgresJsDatabase & { $client: TClient } { - const db = drizzle(client, { schema }); + const db = drizzle(client, { schema, logger: statementCountingLogger }); const tables = db._.schema; if (!tables) return db; const { fullSchema, tableNamesMap } = db._; - const transactions = drizzle(client) as unknown as Transaction; + const transactions = drizzle(client, { logger: statementCountingLogger }) as unknown as Transaction; const dialect = new PgDialect(); const resolved = (value: unknown): PropertyDescriptor => ({ value, writable: true, enumerable: true, configurable: true }); const lazyQuery: PropertyDescriptor = { diff --git a/packages/db/src/statement-count.ts b/packages/db/src/statement-count.ts new file mode 100644 index 00000000000..da0f00f1478 --- /dev/null +++ b/packages/db/src/statement-count.ts @@ -0,0 +1,13 @@ +import type { Logger } from "drizzle-orm"; + +let statements = 0; + +export const statementCountingLogger: Logger = { + logQuery(): void { + statements += 1; + }, +}; + +export function databaseStatementCount(): number { + return statements; +} diff --git a/rules/company-scope-exceptions.mdc b/rules/company-scope-exceptions.mdc index a68dbb9d2e6..7d907af4a09 100644 --- a/rules/company-scope-exceptions.mdc +++ b/rules/company-scope-exceptions.mdc @@ -1,6 +1,6 @@ --- -description: The known instance-wide data is announcement dismissals, the announcement publication-ID registry, personal workspace presentation preferences, and the per-user sidebar company order; these grant no authority, and a new exception must be added to this list. -globs: "packages/db/src/schema/announcement_dismissals.ts,packages/shared/src/announcements.ts,server/src/routes/announcements.ts,server/src/services/announcements.ts,server/src/services/announcement-feed.ts,packages/db/src/schema/user_workspace_preferences.ts,packages/shared/src/workspace-preferences.ts,server/src/routes/workspace-preferences.ts,server/src/services/workspace-preferences.ts,packages/db/src/schema/user_sidebar_preferences.ts,packages/shared/src/types/sidebar-preferences.ts,server/src/routes/sidebar-preferences.ts,server/src/services/sidebar-preferences.ts" +description: The known instance-wide data is announcement dismissals, the announcement publication-ID registry, personal workspace presentation preferences, the per-user sidebar company order, and the instance-admin server metrics snapshot; these grant no authority, and a new exception must be added to this list. +globs: "packages/db/src/schema/announcement_dismissals.ts,packages/shared/src/announcements.ts,server/src/routes/announcements.ts,server/src/services/announcements.ts,server/src/services/announcement-feed.ts,packages/db/src/schema/user_workspace_preferences.ts,packages/shared/src/workspace-preferences.ts,server/src/routes/workspace-preferences.ts,server/src/services/workspace-preferences.ts,packages/db/src/schema/user_sidebar_preferences.ts,packages/shared/src/types/sidebar-preferences.ts,server/src/routes/sidebar-preferences.ts,server/src/services/sidebar-preferences.ts,server/src/routes/instance-server-metrics.ts,server/src/services/server-metrics.ts" alwaysApply: false --- @@ -29,6 +29,13 @@ company scope here. - **The sidebar company order** in `user_sidebar_preferences` is instance-wide by user, because it orders the user's companies. Sidebar preferences inside one company stay company-scoped in `company_user_sidebar_preferences`. +- **The server metrics snapshot** (`GET /api/instance/server-metrics`) describes + one server process: CPU, event-loop load, statement rate, latency + percentiles and missed lease renewals. Company scope does not fit, because the + process serves every company and the numbers cannot be split by company. Only + an instance admin can read it, it is off unless `PAPERCLIP_SERVER_METRICS` is + set, and it returns numbers and a build revision only: no SQL, names, IDs or + hostnames. It persists nothing and changes nothing. ## Requirements diff --git a/scripts/scale-ramp/config.ts b/scripts/scale-ramp/config.ts index 769331262d2..2165cbbe75d 100644 --- a/scripts/scale-ramp/config.ts +++ b/scripts/scale-ramp/config.ts @@ -104,6 +104,16 @@ export function readSecrets(env: NodeJS.ProcessEnv, credentialKey: string | null }; } +export function metricsCredential(options: Pick, secrets: Secrets): string | null { + if (secrets.metricsToken) return secrets.metricsToken; + if (!options.metricsUrl) return null; + try { + return new URL(options.metricsUrl).origin === options.target.baseUrl ? secrets.token : null; + } catch { + return null; + } +} + function adapterOverrides(raw: string | undefined): Record { if (!raw) return {}; let parsed: unknown; diff --git a/scripts/scale-ramp/main.ts b/scripts/scale-ramp/main.ts index 4099781ce2f..db4cab28fe0 100644 --- a/scripts/scale-ramp/main.ts +++ b/scripts/scale-ramp/main.ts @@ -3,7 +3,7 @@ import path from "node:path"; import { createInterface } from "node:readline/promises"; import { cleanupRun, discoverState, type CleanupReport } from "./cleanup.ts"; import { PaperclipClient, Redactor, realClock, type Clock, type Fetcher } from "./client.ts"; -import { FALLBACK_TOKEN_ENV, parseOptions, readSecrets, TOKEN_ENV, type Options } from "./config.ts"; +import { FALLBACK_TOKEN_ENV, metricsCredential, parseOptions, readSecrets, TOKEN_ENV, type Options } from "./config.ts"; import { criteriaTable } from "./criteria.ts"; import { checkSpend, estimateSpend, type SpendEstimate } from "./estimate.ts"; import { Ramp, type RampOutcome } from "./ramp.ts"; @@ -121,7 +121,7 @@ export async function main(argv: readonly string[], deps: MainDeps): Promise interrupted, }); outcome = await ramp.run(); diff --git a/scripts/scale-ramp/ramp.ts b/scripts/scale-ramp/ramp.ts index 2a329474559..9dc68a00faa 100644 --- a/scripts/scale-ramp/ramp.ts +++ b/scripts/scale-ramp/ramp.ts @@ -130,12 +130,42 @@ function rows(body: unknown): Json[] { return []; } -export function serverMetrics(samples: ReadonlyArray>): ServerMetrics { +type Sample = Record; +const processOf = (sample: Sample) => typeof sample.processId === "string" ? sample.processId : ""; + +function counterBaseline(first: Sample, earlier: readonly Sample[], runStartedAt: number | null, counter: string): number { + const processId = processOf(first); + for (let index = earlier.length - 1; index >= 0; index -= 1) { + const value = num(earlier[index]![counter]); + if (processOf(earlier[index]!) === processId && value !== null) return value; + } + const uptimeSec = num(first.uptimeSec); + const sampledAt = num(first.sampledAt); + if (runStartedAt !== null && uptimeSec !== null && sampledAt !== null && sampledAt - uptimeSec * 1_000 > runStartedAt) return 0; + return num(first[counter]) ?? 0; +} + +function counterGrowth(samples: readonly Sample[], earlier: readonly Sample[], runStartedAt: number | null, counter: string): number | null { + const byProcess = new Map(); + for (const sample of samples) { + if (num(sample[counter]) === null) continue; + const processId = processOf(sample); + byProcess.set(processId, [...byProcess.get(processId) ?? [], sample]); + } + if (byProcess.size === 0) return null; + let growth = 0; + for (const processSamples of byProcess.values()) { + const last = num(processSamples[processSamples.length - 1]![counter])!; + growth += Math.max(0, last - counterBaseline(processSamples[0]!, earlier, runStartedAt, counter)); + } + return growth; +} + +export function serverMetrics(samples: ReadonlyArray, history: { earlier?: readonly Sample[]; runStartedAt?: number | null } = {}): ServerMetrics { const max = (key: string) => { const values = samples.map(sample => num(sample[key])).filter((value): value is number => value !== null); return values.length ? Math.max(...values) : null; }; - const missed = samples.map(sample => num(sample.leaseRenewalsMissed)).filter((value): value is number => value !== null); return { mainThreadCpuMax: max("mainThreadCpu"), eventLoopUtilizationMax: max("eventLoopUtilization"), @@ -144,7 +174,7 @@ export function serverMetrics(samples: ReadonlyArray>): poolWaitP99MsMax: max("poolWaitP99Ms"), runEventLatenessP99MsMax: max("runEventLatenessP99Ms"), agentApiPreHandlerP99MsMax: max("agentApiPreHandlerP99Ms"), - leaseRenewalsMissed: missed.length ? missed[missed.length - 1]! - missed[0]! : null, + leaseRenewalsMissed: counterGrowth(samples, history.earlier ?? [], history.runStartedAt ?? null, "leaseRenewalsMissed"), samples: samples.length, }; } @@ -467,7 +497,10 @@ export class Ramp { firstActivityMs: distribution(accumulator.firstActivityMs), runDurationMs: distribution(runs.flatMap(run => run.startedAt !== null && run.finishedAt !== null ? [run.finishedAt - run.startedAt] : [])), board: { requests: accumulator.boardRequests, failures: accumulator.boardFailures, ms: distribution(accumulator.boardMs) }, - server: serverMetrics(accumulator.serverSamples), + server: serverMetrics(accumulator.serverSamples, { + earlier: this.accumulators.slice(0, stepIndex).flatMap(earlier => earlier.serverSamples), + runStartedAt: this.accumulators[0]?.startedAt ?? null, + }), recordedSpendUsd: Math.round(spend * 100) / 100, }; } diff --git a/scripts/scale-ramp/scale-ramp.test.ts b/scripts/scale-ramp/scale-ramp.test.ts index 8751bde83ad..6a34655e617 100644 --- a/scripts/scale-ramp/scale-ramp.test.ts +++ b/scripts/scale-ramp/scale-ramp.test.ts @@ -1,16 +1,17 @@ import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { after, describe, it } from "node:test"; import { PaperclipClient, Redactor } from "./client.ts"; -import { parseOptions, QUALIFIED_OPENCODE_MODEL } from "./config.ts"; +import { metricsCredential, parseOptions, QUALIFIED_OPENCODE_MODEL, readSecrets } from "./config.ts"; import { evaluate, type StepMetrics } from "./criteria.ts"; import { checkSpend, estimateSpend } from "./estimate.ts"; import { FakePaperclip, VirtualClock, type FakeConfig } from "./fake-paperclip.ts"; import { classifyHost, resolveTarget, TargetRefused } from "./guard.ts"; import { EXIT, main, type MainDeps } from "./main.ts"; -import { Ramp } from "./ramp.ts"; +import { Ramp, serverMetrics } from "./ramp.ts"; import { verdictOf, type RampResult } from "./report.ts"; import { ensureAgents, prepareCompany } from "./setup.ts"; import { distribution } from "./stats.ts"; @@ -246,6 +247,75 @@ describe("criteria", () => { }); }); +describe("server metrics endpoint contract", () => { + const fixture = JSON.parse(readFileSync(new URL("./server-metrics.fixture.json", import.meta.url), "utf8")) as Record; + const SERVER_METRICS_URL = `${ORIGIN}/api/instance/server-metrics`; + + it("parses every finish-line field of the server's own serialized response except pool wait, which the server does not measure", () => { + const parsed = serverMetrics([fixture, { ...fixture, leaseRenewalsMissed: 3 }]); + assert.deepEqual(parsed, { + mainThreadCpuMax: fixture.mainThreadCpu, eventLoopUtilizationMax: fixture.eventLoopUtilization, eventLoopDelayP99MsMax: fixture.eventLoopDelayP99Ms, + statementsPerSecondMax: fixture.statementsPerSecond, poolWaitP99MsMax: null, runEventLatenessP99MsMax: fixture.runEventLatenessP99Ms, + agentApiPreHandlerP99MsMax: fixture.agentApiPreHandlerP99Ms, leaseRenewalsMissed: 2, samples: 2, + }); + assert.equal("poolWaitP99Ms" in fixture, false); + }); + + it("counts missed lease renewals per server process, so samples from several processes never cancel out", () => { + const parsed = serverMetrics([ + { processId: "a", leaseRenewalsMissed: 5 }, { processId: "b", leaseRenewalsMissed: 0 }, + { processId: "a", leaseRenewalsMissed: 6 }, { processId: "b", leaseRenewalsMissed: 2 }, { processId: "a", leaseRenewalsMissed: 6 }, + ]); + assert.equal(parsed.leaseRenewalsMissed, 3); + }); + + it("counts the full counter of a process that started during the run, so a restart cannot hide a missed renewal", () => { + const runStartedAt = 1_000_000; + const earlier = [{ processId: "a", uptimeSec: 3_600, sampledAt: runStartedAt + 10_000, leaseRenewalsMissed: 4 }]; + const parsed = serverMetrics([ + { processId: "a", uptimeSec: 3_630, sampledAt: runStartedAt + 40_000, leaseRenewalsMissed: 4 }, + { processId: "b", uptimeSec: 20, sampledAt: runStartedAt + 70_000, leaseRenewalsMissed: 1 }, + { processId: "b", uptimeSec: 35, sampledAt: runStartedAt + 85_000, leaseRenewalsMissed: 1 }, + ], { earlier, runStartedAt }); + assert.equal(parsed.leaseRenewalsMissed, 1); + }); + + it("keeps a process's counter baseline from earlier steps", () => { + const runStartedAt = 1_000_000; + const earlier = [ + { processId: "a", uptimeSec: 3_600, sampledAt: runStartedAt + 10_000, leaseRenewalsMissed: 4 }, + { processId: "a", uptimeSec: 3_650, sampledAt: runStartedAt + 60_000, leaseRenewalsMissed: 6 }, + ]; + const parsed = serverMetrics([{ processId: "a", uptimeSec: 3_700, sampledAt: runStartedAt + 110_000, leaseRenewalsMissed: 7 }], + { earlier, runStartedAt }); + assert.equal(parsed.leaseRenewalsMissed, 1); + assert.equal(serverMetrics([{ processId: "a", uptimeSec: 3_700, sampledAt: runStartedAt + 110_000, leaseRenewalsMissed: 7 }], { runStartedAt }) + .leaseRenewalsMissed, 0, "a process that predates the run and is first seen now starts from its first sample"); + }); + + it("sends the board token to a metrics URL on the target origin, and no token to another origin", () => { + const options = parseOptions(["--base-url", ORIGIN, "--target-kind", "test-stack"], { PAPERCLIP_SCALE_METRICS_URL: SERVER_METRICS_URL }, 0); + assert.equal(metricsCredential(options, readSecrets({ PAPERCLIP_SCALE_TOKEN: TOKEN }, null)), TOKEN); + assert.equal(metricsCredential(options, readSecrets({ PAPERCLIP_SCALE_TOKEN: TOKEN, PAPERCLIP_SCALE_METRICS_TOKEN: METRICS_TOKEN }, null)), METRICS_TOKEN); + const elsewhere = parseOptions(["--base-url", ORIGIN, "--target-kind", "test-stack"], { PAPERCLIP_SCALE_METRICS_URL: METRICS_URL }, 0); + assert.equal(metricsCredential(elsewhere, readSecrets({ PAPERCLIP_SCALE_TOKEN: TOKEN }, null)), null); + }); + + it("measures the server criteria from the server metrics endpoint with the board token in the authorization header", async () => { + const out = await tempDir(); + const h = harness({ metricsUrl: SERVER_METRICS_URL, metrics: () => fixture }, { PAPERCLIP_SCALE_METRICS_URL: SERVER_METRICS_URL }); + await main(baseArgs(out, ["--steps", "2"]), h.deps); + const result = await readResult(out); + const metricsRequests = h.fake.requests.filter(request => request.url === SERVER_METRICS_URL); + assert.ok(metricsRequests.length > 0); + assert.ok(metricsRequests.every(request => request.authorization === `Bearer ${TOKEN}`)); + assert.equal(result.sources.serverMetrics, "PAPERCLIP_SCALE_METRICS_URL"); + const statuses = Object.fromEntries(result.steps[0]!.criteriaFinal.map(criterion => [criterion.id, criterion.status])); + for (const id of ["main_thread_cpu", "run_event_lateness_p99", "agent_api_pre_handler_p99"]) assert.equal(statuses[id], "pass", id); + assert.equal(statuses.lease_renewals_missed_zero, "pass"); + }); +}); + describe("ramp against the fake API", () => { it("ramps every step, measures it, passes with a metrics endpoint and cleans up", async () => { const out = await tempDir(); diff --git a/scripts/scale-ramp/server-metrics.fixture.json b/scripts/scale-ramp/server-metrics.fixture.json new file mode 100644 index 00000000000..6897aaedfc5 --- /dev/null +++ b/scripts/scale-ramp/server-metrics.fixture.json @@ -0,0 +1,27 @@ +{ + "schema": "paperclip.server-metrics.v1", + "processId": "a1b2c3d4e5f60718", + "buildRevision": "0123456789abcdef0123456789abcdef01234567", + "uptimeSec": 160, + "sampledAt": "2026-11-14T12:01:00.000Z", + "windowMs": 50000, + "mainThreadCpu": 0.42, + "processCpu": 0.55, + "eventLoopUtilization": 0.42, + "eventLoopDelayP99Ms": 0, + "statementsPerSecond": 180, + "runEventLatenessP99Ms": 120.063, + "runEventLatenessSamples": 50, + "agentApiPreHandlerP99Ms": 15.007, + "agentApiPreHandlerSamples": 50, + "leaseRenewalsMissed": 1, + "dedicatedConnections": { + "limit": 24, + "inUse": 3, + "peakInUse": 9, + "waiting": 1, + "queued": 12, + "overLimit": 0, + "nestedOverLimit": 0 + } +} diff --git a/server/src/__tests__/instance-server-metrics-routes.test.ts b/server/src/__tests__/instance-server-metrics-routes.test.ts new file mode 100644 index 00000000000..71cd3aaf24f --- /dev/null +++ b/server/src/__tests__/instance-server-metrics-routes.test.ts @@ -0,0 +1,140 @@ +import { createHash, randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agentApiKeys, agents, authUsers, boardApiKeys, companies, companyMemberships, createDb, databaseStatementCount, + instanceUserRoles } from "@paperclipai/db"; +import { createApp } from "../app.js"; +import { activeServerMetrics, recordMissedLeaseRenewal, recordRunEventCommitted } from "../services/server-metrics.js"; +import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; +import { createStorageService } from "../storage/service.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const describePostgres = support.supported ? describe : describe.skip; +const PATH = "/api/instance/server-metrics"; +const FIELDS = [ + "schema", "processId", "buildRevision", "uptimeSec", "sampledAt", "windowMs", "mainThreadCpu", "processCpu", + "eventLoopUtilization", "eventLoopDelayP99Ms", "statementsPerSecond", "runEventLatenessP99Ms", "runEventLatenessSamples", + "agentApiPreHandlerP99Ms", "agentApiPreHandlerSamples", "leaseRenewalsMissed", "dedicatedConnections", +].sort(); + +type Db = ReturnType; +type App = Awaited>; + +describePostgres("instance server metrics route", () => { + let database: Awaited>; + let db: Db; + let directory: string; + const hash = (token: string) => createHash("sha256").update(token).digest("hex"); + const ids = { companyId: randomUUID(), agentId: randomUUID(), adminId: randomUUID(), memberId: randomUUID() }; + const keys = { admin: `pcp_board_${randomUUID()}`, member: `pcp_board_${randomUUID()}`, agent: `pcp_agent_${randomUUID()}` }; + + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("instance-server-metrics-"); + db = createDb(database.connectionString); + directory = await mkdtemp(join(tmpdir(), "instance-server-metrics-storage-")); + await db.insert(companies).values({ id: ids.companyId, name: "Metrics company", issuePrefix: "MET" }); + await db.insert(authUsers).values([ids.adminId, ids.memberId].map(id => ({ id, name: "Person", email: `${id}@example.com`, + createdAt: new Date(), updatedAt: new Date() }))); + await db.insert(instanceUserRoles).values({ userId: ids.adminId, role: "instance_admin" }); + await db.insert(agents).values({ id: ids.agentId, companyId: ids.companyId, name: "Metrics agent", adapterType: "process", + status: "idle", permissions: {}, runtimeConfig: { heartbeat: { wakeOnDemand: false, intervalSec: 0 } } }); + await db.insert(companyMemberships).values([ + { companyId: ids.companyId, principalType: "user", principalId: ids.memberId, membershipRole: "owner", status: "active" }, + { companyId: ids.companyId, principalType: "agent", principalId: ids.agentId, membershipRole: "member", status: "active" }, + ]); + await db.insert(boardApiKeys).values([ + { userId: ids.adminId, name: "Admin", keyHash: hash(keys.admin) }, + { userId: ids.memberId, name: "Member", keyHash: hash(keys.member) }, + ]); + await db.insert(agentApiKeys).values({ companyId: ids.companyId, agentId: ids.agentId, name: "Agent", keyHash: hash(keys.agent), + responsibleUserId: ids.memberId }); + }, 120_000); + + afterAll(async () => { + await db?.$client.end(); + await database?.cleanup(); + if (directory) await rm(directory, { recursive: true, force: true }); + }); + + async function serve(serverMetricsEnabled: boolean, body: (call: (path: string, token?: string) => Promise<{ status: number; body: any; text: string }>) => Promise) { + const app: App = await createApp(db, { + uiMode: "none", serverPort: 0, bindHost: "127.0.0.1", allowedHostnames: ["127.0.0.1"], + deploymentMode: "authenticated", deploymentExposure: "private", authReady: true, companyDeletionEnabled: false, + managedPluginAutoInstall: [], storageService: createStorageService(createLocalDiskStorageProvider(directory)), + decisionServiceOptions: { wakeOriginAgent: async () => null }, serverMetricsEnabled, + }); + await app.locals.bundledPluginsStartup; + const server = app.listen(0, "127.0.0.1"); + try { + await new Promise(resolve => server.once("listening", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("No HTTP test address"); + const call = async (path: string, token?: string) => { + const response = await fetch(`http://127.0.0.1:${address.port}${path}`, { headers: token ? { authorization: `Bearer ${token}` } : {} }); + const text = await response.text(); + let parsed: unknown = null; + try { parsed = JSON.parse(text); } catch { parsed = null; } + return { status: response.status, body: parsed, text }; + }; + return await body(call); + } finally { + await app.locals.paperclipShutdown?.(); + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + } + } + + it("serves numbers to an instance admin only, and they move with load", async () => { + await serve(true, async (call) => { + expect(activeServerMetrics()).not.toBeNull(); + expect([401, 403]).toContain((await call(PATH)).status); + expect((await call(PATH, keys.member)).status).toBe(403); + expect((await call(PATH, keys.agent)).status).toBe(403); + + const first = await call(PATH, keys.admin); + expect(first.status).toBe(200); + expect(Object.keys(first.body).sort()).toEqual(FIELDS); + expect(first.body).toMatchObject({ schema: "paperclip.server-metrics.v1", processId: expect.stringMatching(/^[0-9a-f]{16}$/) }); + expect(first.body.buildRevision === null || /^[0-9a-f]{40}$/.test(first.body.buildRevision)).toBe(true); + for (const field of ["uptimeSec", "windowMs", "processCpu", "eventLoopUtilization", "eventLoopDelayP99Ms", "statementsPerSecond", + "runEventLatenessSamples", "agentApiPreHandlerSamples", "leaseRenewalsMissed"]) { + expect(typeof first.body[field], field).toBe("number"); + } + expect(first.body.eventLoopUtilization).toBeGreaterThanOrEqual(0); + expect(first.body.dedicatedConnections).toMatchObject({ limit: expect.any(Number), inUse: expect.any(Number), waiting: expect.any(Number), + queued: expect.any(Number) }); + expect(first.body.eventLoopUtilization).toBeLessThanOrEqual(1); + + const statementsBefore = databaseStatementCount(); + for (let index = 0; index < 5; index += 1) expect((await call("/api/agents/me", keys.agent)).status).toBe(200); + expect(databaseStatementCount()).toBeGreaterThan(statementsBefore); + recordRunEventCommitted(new Date(Date.now() - 750).toISOString(), Date.now()); + recordMissedLeaseRenewal(); + + const second = await call(PATH, keys.admin); + expect(second.status).toBe(200); + expect(second.body.processId).toBe(first.body.processId); + expect(second.body.agentApiPreHandlerSamples).toBeGreaterThanOrEqual(first.body.agentApiPreHandlerSamples + 5); + expect(second.body.agentApiPreHandlerP99Ms).toBeGreaterThan(0); + expect(second.body.statementsPerSecond).toBeGreaterThan(0); + expect(second.body.runEventLatenessSamples).toBe(first.body.runEventLatenessSamples + 1); + expect(second.body.runEventLatenessP99Ms).toBeGreaterThanOrEqual(700); + expect(second.body.leaseRenewalsMissed).toBe(first.body.leaseRenewalsMissed + 1); + for (const secret of [...Object.values(ids), ...Object.values(keys)]) expect(second.text).not.toContain(secret); + expect(second.text).not.toMatch(/select|insert|update|from\s/i); + }); + expect(activeServerMetrics()).toBeNull(); + }, 240_000); + + it("is absent and collects nothing when the kill switch is off", async () => { + await serve(false, async (call) => { + expect(activeServerMetrics()).toBeNull(); + expect((await call(PATH, keys.admin)).status).toBe(404); + expect((await call(PATH, keys.member)).status).toBe(404); + expect((await call("/api/agents/me", keys.agent)).status).toBe(200); + expect(activeServerMetrics()).toBeNull(); + }); + }, 240_000); +}); diff --git a/server/src/__tests__/openapi-routes.test.ts b/server/src/__tests__/openapi-routes.test.ts index d4b6e5b8e77..399b53fc554 100644 --- a/server/src/__tests__/openapi-routes.test.ts +++ b/server/src/__tests__/openapi-routes.test.ts @@ -76,6 +76,7 @@ const apiPrefixes: Record = { "inbox-agent-policy.ts": "/api", "inbox-dismissals.ts": "/api", "instance-database-backups.ts": "/api", + "instance-server-metrics.ts": "/api", "instance-settings.ts": "/api", "issues.ts": "/api", "issue-tree-control.ts": "/api", diff --git a/server/src/__tests__/server-metrics.test.ts b/server/src/__tests__/server-metrics.test.ts new file mode 100644 index 00000000000..e0c20ce0b62 --- /dev/null +++ b/server/src/__tests__/server-metrics.test.ts @@ -0,0 +1,196 @@ +import { readFileSync, writeFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import type { EventLoopUtilization } from "node:perf_hooks"; +import { + activeServerMetrics, + recordMissedLeaseRenewal, + recordRunEventCommitted, + ServerMetricsCollector, + serverMetricsEnabledFromEnv, + startServerMetrics, + stopServerMetrics, + type ServerMetricsProbes, +} from "../services/server-metrics.js"; + +const FIXTURE = fileURLToPath(new URL("../../../scripts/scale-ramp/server-metrics.fixture.json", import.meta.url)); + +function fakeProbes() { + const state = { now: 0, wall: Date.parse("2026-11-14T12:00:00.000Z"), idle: 0, active: 0, cpuMicros: 0, mainMs: 0, statements: 0, uptime: 100 }; + const elu = (): EventLoopUtilization => ({ idle: state.idle, active: state.active, utilization: 0 }); + const probes: ServerMetricsProbes = { + now: () => state.now, + wallNow: () => state.wall, + eventLoopUtilization: (current?: EventLoopUtilization, previous?: EventLoopUtilization) => { + if (!current) return elu(); + const idle = current.idle - (previous?.idle ?? 0); + const active = current.active - (previous?.active ?? 0); + return { idle, active, utilization: idle + active > 0 ? active / (idle + active) : 0 }; + }, + processCpuMicros: () => state.cpuMicros, + mainThreadCpuMs: () => state.mainMs, + statementCount: () => state.statements, + uptimeSec: () => state.uptime, + dedicatedConnections: () => ({ limit: 24, inUse: 3, peakInUse: 9, waiting: 1, queued: 12, overLimit: 0, nestedOverLimit: 0 }), + }; + const advance = (ms: number, load: { busy: number; cpu?: number; statementsPerSecond?: number }) => { + state.now += ms; + state.wall += ms; + state.uptime += ms / 1000; + state.active += ms * load.busy; + state.idle += ms * (1 - load.busy); + state.mainMs += ms * load.busy; + state.cpuMicros += ms * (load.cpu ?? load.busy) * 1000; + state.statements += (ms / 1000) * (load.statementsPerSecond ?? 0); + }; + return { probes, advance }; +} + +const collectors: ServerMetricsCollector[] = []; +function collector(probes: ServerMetricsProbes) { + const created = new ServerMetricsCollector({ probes, bucketMs: 10_000, buckets: 6, buildRevision: "0123456789abcdef0123456789abcdef01234567", + monitorEventLoop: false, processId: "a1b2c3d4e5f60718" }); + collectors.push(created); + return created; +} + +afterEach(() => { + for (const created of collectors.splice(0)) created.stop(); + const active = activeServerMetrics(); + if (active) stopServerMetrics(active); +}); + +describe("server metrics collector", () => { + it("reports rates and utilization over the sliding window, dropping buckets older than the window", () => { + const { probes, advance } = fakeProbes(); + const metrics = collector(probes); + for (let bucket = 0; bucket < 6; bucket += 1) { + advance(10_000, { busy: 0.9, statementsPerSecond: 500 }); + metrics.rotate(); + } + const busy = metrics.snapshot(); + expect(busy.windowMs).toBe(50_000); + expect(busy.eventLoopUtilization).toBeCloseTo(0.9, 4); + expect(busy.mainThreadCpu).toBeCloseTo(0.9, 4); + expect(busy.statementsPerSecond).toBeCloseTo(500, 2); + + for (let bucket = 0; bucket < 6; bucket += 1) { + advance(10_000, { busy: 0.1, statementsPerSecond: 20 }); + metrics.rotate(); + } + const quiet = metrics.snapshot(); + expect(quiet.eventLoopUtilization).toBeCloseTo(0.1, 4); + expect(quiet.statementsPerSecond).toBeCloseTo(20, 2); + }); + + it("expires buckets by elapsed time when the rotation timer fires late", () => { + const { probes, advance } = fakeProbes(); + const metrics = collector(probes); + for (let bucket = 0; bucket < 6; bucket += 1) { + advance(10_000, { busy: 0.1, statementsPerSecond: 10 }); + metrics.rotate(); + } + metrics.recordRunEventLatenessMs(5_000); + advance(35_000, { busy: 0.5, statementsPerSecond: 100 }); + metrics.rotate(); + const late = metrics.snapshot(); + expect(late.windowMs).toBe(55_000); + expect(late.statementsPerSecond).toBeCloseTo((20 * 10 + 35 * 100) / 55, 2); + expect(late.runEventLatenessSamples).toBe(1); + }); + + it("reports the real window after a block longer than the window, and the block shows in it", () => { + const { probes, advance } = fakeProbes(); + const metrics = collector(probes); + for (let bucket = 0; bucket < 6; bucket += 1) { + advance(10_000, { busy: 0.1 }); + metrics.rotate(); + } + metrics.recordRunEventLatenessMs(20); + advance(300_000, { busy: 1 }); + metrics.rotate(); + const blocked = metrics.snapshot(); + expect(blocked.windowMs).toBe(300_000); + expect(blocked.eventLoopUtilization).toBeCloseTo(1, 4); + expect(blocked.runEventLatenessSamples).toBe(1); + advance(10_000, { busy: 0.2 }); + metrics.rotate(); + expect(metrics.snapshot()).toMatchObject({ windowMs: 10_000, runEventLatenessSamples: 0 }); + for (let bucket = 0; bucket < 5; bucket += 1) { + advance(10_000, { busy: 0.2 }); + metrics.rotate(); + } + const recovered = metrics.snapshot(); + expect(recovered.windowMs).toBe(50_000); + expect(recovered.eventLoopUtilization).toBeCloseTo(0.2, 4); + }); + + it("reports latency percentiles only for the window and absent when nothing was recorded", () => { + const { probes, advance } = fakeProbes(); + const metrics = collector(probes); + expect(metrics.snapshot()).toMatchObject({ runEventLatenessP99Ms: null, runEventLatenessSamples: 0, + agentApiPreHandlerP99Ms: null, agentApiPreHandlerSamples: 0 }); + for (let index = 0; index < 100; index += 1) metrics.recordRunEventLatenessMs(index < 98 ? 20 : 2_000); + metrics.recordAgentApiPreHandlerMs(3.5); + const recorded = metrics.snapshot(); + expect(recorded.runEventLatenessSamples).toBe(100); + expect(recorded.runEventLatenessP99Ms).toBeGreaterThan(1_900); + expect(recorded.agentApiPreHandlerP99Ms).toBeCloseTo(3.5, 1); + for (let bucket = 0; bucket < 6; bucket += 1) { + advance(10_000, { busy: 0 }); + metrics.rotate(); + } + expect(metrics.snapshot()).toMatchObject({ runEventLatenessP99Ms: null, runEventLatenessSamples: 0, agentApiPreHandlerSamples: 0 }); + }); + + it("counts missed lease renewals since start and never resets them with the window", () => { + const { probes, advance } = fakeProbes(); + const metrics = collector(probes); + metrics.recordMissedLeaseRenewal(); + metrics.recordMissedLeaseRenewal(); + for (let bucket = 0; bucket < 8; bucket += 1) { + advance(10_000, { busy: 0 }); + metrics.rotate(); + } + expect(metrics.snapshot().leaseRenewalsMissed).toBe(2); + }); + + it("records into the process collector only while it runs", () => { + recordRunEventCommitted(new Date(Date.now() - 500).toISOString(), Date.now()); + recordMissedLeaseRenewal(); + expect(activeServerMetrics()).toBeNull(); + const started = startServerMetrics(); + expect(startServerMetrics()).toBe(started); + recordRunEventCommitted(new Date(Date.now() - 500).toISOString(), Date.now()); + recordRunEventCommitted("not a time", Date.now()); + recordMissedLeaseRenewal(); + const snapshot = started.snapshot(); + expect(snapshot.runEventLatenessSamples).toBe(1); + expect(snapshot.runEventLatenessP99Ms).toBeGreaterThanOrEqual(450); + expect(snapshot.leaseRenewalsMissed).toBe(1); + stopServerMetrics(started); + expect(activeServerMetrics()).toBeNull(); + }); + + it("reads the kill switch strictly and defaults it off", () => { + expect(serverMetricsEnabledFromEnv({})).toBe(false); + expect(serverMetricsEnabledFromEnv({ PAPERCLIP_SERVER_METRICS: "true" })).toBe(true); + expect(serverMetricsEnabledFromEnv({ PAPERCLIP_SERVER_METRICS: "0" })).toBe(false); + expect(() => serverMetricsEnabledFromEnv({ PAPERCLIP_SERVER_METRICS: "yes" })).toThrow(/PAPERCLIP_SERVER_METRICS/); + }); + + it("serializes exactly the fixture the scale-ramp harness parses", () => { + const { probes, advance } = fakeProbes(); + const metrics = collector(probes); + for (let bucket = 0; bucket < 6; bucket += 1) { + advance(10_000, { busy: 0.42, cpu: 0.55, statementsPerSecond: 180 }); + metrics.rotate(); + } + for (let index = 0; index < 50; index += 1) metrics.recordRunEventLatenessMs(120); + for (let index = 0; index < 50; index += 1) metrics.recordAgentApiPreHandlerMs(15); + metrics.recordMissedLeaseRenewal(); + const snapshot = JSON.parse(JSON.stringify(metrics.snapshot())); + if (process.env.UPDATE_SERVER_METRICS_FIXTURE === "1") writeFileSync(FIXTURE, `${JSON.stringify(snapshot, null, 2)}\n`); + expect(snapshot).toEqual(JSON.parse(readFileSync(FIXTURE, "utf8"))); + }); +}); diff --git a/server/src/app.ts b/server/src/app.ts index a3682ea2a26..ff1b4d6a6df 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -11,7 +11,7 @@ import { finishServiceShutdown } from "./shutdown.js"; import { promptInjectionPreventionRoutes } from "./routes/prompt-injection-prevention.js"; import { voiceSessionRoutes, voiceWebhookRoutes } from "./routes/voice-sessions.js"; import { eq } from "drizzle-orm"; -import { chatEndpoints } from "@paperclipai/db"; +import { chatEndpoints, dedicatedDbConnectionStats } from "@paperclipai/db"; import { fastResponseRoutes } from "./routes/fast-responses.js"; import { fastResponseService } from "./services/fast-responses.js"; import { createDeliveryWorkCoordinator } from "./services/delivery-work-coordinator.js"; @@ -253,6 +253,14 @@ import { import { COMPANY_IMPORT_API_PATH } from "./routes/company-import-paths.js"; import { apiCompression } from "./middleware/api-compression.js"; import { apiServerTiming } from "./middleware/api-server-timing.js"; +import { instanceServerMetricsRoutes } from "./routes/instance-server-metrics.js"; +import { + serverMetricsAgentPreHandler, + serverMetricsArrival, + serverMetricsEnabledFromEnv, + startServerMetrics, + stopServerMetrics, +} from "./services/server-metrics.js"; import { chatWebhookBodyParser } from "./middleware/chat-webhook-body.js"; import { createChatWebhookDiagnostics } from "./services/chat-webhook-diagnostics.js"; @@ -491,9 +499,12 @@ export async function createApp( managedPluginAutoInstall?: readonly string[] | null; /** Test override for the bundled plugin catalog root. */ bundledPluginCatalogRoot?: string; + serverMetricsEnabled?: boolean; }, ) { const app = express(); + const serverMetrics = (opts.serverMetricsEnabled ?? serverMetricsEnabledFromEnv(process.env)) ? startServerMetrics(() => dedicatedDbConnectionStats(db)) : null; + if (serverMetrics) app.use("/api", serverMetricsArrival()); // First, so the reported server time includes admission and database waits. app.use("/api", apiServerTiming()); app.use(databaseOperationMiddleware); @@ -886,6 +897,7 @@ export async function createApp( if (opts.databaseBackupService) { api.use(instanceDatabaseBackupRoutes(opts.databaseBackupService)); } + if (serverMetrics) api.use(instanceServerMetricsRoutes(serverMetrics)); const pluginRegistry = pluginRegistryService(db); const eventBus = createPluginEventBus(); setPluginEventBus(eventBus); @@ -1084,6 +1096,7 @@ export async function createApp( api.use(museBoardRoutes(db, opts.authPublicBaseUrl)); api.use(museQualificationRoutes(db)); + if (serverMetrics) app.use("/api", serverMetricsAgentPreHandler(serverMetrics)); app.use("/api", api); app.use("/api", (_req, res) => { res.status(404).json({ error: "API route not found" }); @@ -1448,6 +1461,7 @@ export async function createApp( // caller stops the database and the provider. A lease release that // fails stays a durable record for the startup reaper. await setupTokenLoginService?.shutdown(); + if (serverMetrics) stopServerMetrics(serverMetrics); }); }); return appServicesShutdown; diff --git a/server/src/routes/instance-server-metrics.ts b/server/src/routes/instance-server-metrics.ts new file mode 100644 index 00000000000..247fe952e5f --- /dev/null +++ b/server/src/routes/instance-server-metrics.ts @@ -0,0 +1,15 @@ +import { Router } from "express"; +import type { ServerMetricsCollector } from "../services/server-metrics.js"; +import { assertInstanceAdmin } from "./authz.js"; + +export function instanceServerMetricsRoutes(collector: ServerMetricsCollector) { + const router = Router(); + + router.get("/instance/server-metrics", (req, res) => { + assertInstanceAdmin(req); + res.set("Cache-Control", "no-store"); + res.json(collector.snapshot()); + }); + + return router; +} diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 3803834b536..8762d89f512 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -1642,6 +1642,7 @@ const INSTANCE_ADMIN_OPERATIONS = new Set([ "POST /api/companies", "POST /api/plugins/install", "POST /api/instance/database-backups", + "GET /api/instance/server-metrics", "POST /api/admin/users/{userId}/promote-instance-admin", "POST /api/admin/users/{userId}/demote-instance-admin", "PUT /api/admin/users/{userId}/company-access", @@ -10391,6 +10392,17 @@ registry.registerPath({ responses: { 201: r.ok(), 401: r.unauthorized, 403: r.forbidden }, }); +// ─── Instance server metrics ────────────────────────────────────────────────── + +registry.registerPath({ + method: "get", + path: "/api/instance/server-metrics", + tags: ["instance"], + summary: "Get this server process's load metrics", + description: "Instance admins only: agent credentials and company members receive 403. Returns numbers for the one server process that answers: CPU, event-loop utilization and delay, statements per second, run event lateness and agent API pre-handler p99, missed lease renewals and dedicated database connection use, over about the last minute (windowMs). It returns 404 unless the operator set PAPERCLIP_SERVER_METRICS=true.", + responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden, 404: r.notFound }, +}); + // ─── LLM text endpoints ─────────────────────────────────────────────────────── registry.registerPath({ diff --git a/server/src/services/native-runtime/native-execution-lease.test.ts b/server/src/services/native-runtime/native-execution-lease.test.ts index 26145d855f5..046d0a3a5bc 100644 --- a/server/src/services/native-runtime/native-execution-lease.test.ts +++ b/server/src/services/native-runtime/native-execution-lease.test.ts @@ -3,6 +3,7 @@ import type { SQL } from "drizzle-orm"; import { PgDialect } from "drizzle-orm/pg-core"; import type { Db } from "@paperclipai/db"; import { startNativeSessionExecutionLeaseRenewal } from "./native-execution-lease.js"; +import { startServerMetrics, stopServerMetrics } from "../server-metrics.js"; const dialect = new PgDialect(); @@ -134,6 +135,24 @@ describe("native execution renewal ownership", () => { expect(returning).toHaveBeenCalledTimes(1); expect(vi.getTimerCount()).toBe(0); }); + it("counts a renewal deadline that passes as a missed lease renewal for the server metrics", async () => { + const metrics = startServerMetrics(); + try { + vi.useFakeTimers(); + const { input, returning } = fixture(); + let complete!: (value: { runId: string }[]) => void; + returning.mockImplementationOnce(() => new Promise(resolve => { complete = resolve; })); + const guard = startNativeSessionExecutionLeaseRenewal(input); + await vi.advanceTimersByTimeAsync(30_000); + expect(metrics.snapshot().leaseRenewalsMissed).toBe(0); + await vi.advanceTimersByTimeAsync(30_000); + expect(metrics.snapshot().leaseRenewalsMissed).toBe(1); + complete([{ runId: "run" }]); + await expect(guard.stop()).rejects.toThrow("native_session_lease_renewal_deadline"); + } finally { + stopServerMetrics(metrics); + } + }); it("hands preparation renewal to the same owner's executor at the next attempt", async () => { vi.useFakeTimers(); const { input, returning } = fixture(); diff --git a/server/src/services/native-runtime/native-execution-lease.ts b/server/src/services/native-runtime/native-execution-lease.ts index a46820364c3..9ee78b6e4b9 100644 --- a/server/src/services/native-runtime/native-execution-lease.ts +++ b/server/src/services/native-runtime/native-execution-lease.ts @@ -5,6 +5,7 @@ import { currentNativeControllerIdentity, type NativeControllerIdentity } from " import { OPERATOR_ATTESTED_CONTROLLER_HOST } from "./native-controller-host-recovery.js"; import { NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE } from "./native-runner-ownership.js"; import { REMOTE_CONTROLLER_LEASE_MS } from "./native-runner-recovery-contract.js"; +import { recordMissedLeaseRenewal } from "../server-metrics.js"; export const NATIVE_SESSION_EXECUTION_LEASE_TTL_MS = 20 * 60_000; const NATIVE_SESSION_EXECUTION_LEASE_RENEW_INTERVAL_MS = 5 * 60_000; @@ -613,7 +614,10 @@ export function startNativeSessionExecutionLeaseRenewal(input: RenewalInput): Na state!.loss = Promise.resolve().then(state!.input.onLost).catch(() => undefined); }; const deadline = (delay: number) => { - const timer = setTimeout(() => lose(new Error("native_session_lease_renewal_deadline")), Math.max(0, delay)); + const timer = setTimeout(() => { + recordMissedLeaseRenewal(); + lose(new Error("native_session_lease_renewal_deadline")); + }, Math.max(0, delay)); timer.unref?.(); return timer; }; diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts index bbda1e48c8b..2975c229823 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts @@ -2,6 +2,7 @@ import { nativeResponsePresentationWork } from "./native-response-presentation-w import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; import { plugins } from "@paperclipai/db"; import { readTaskPanelActivations } from "../plugin-task-panels.js"; +import { startServerMetrics, stopServerMetrics } from "../server-metrics.js"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { and, asc, eq, sql } from "drizzle-orm"; import { createHash, generateKeyPairSync, randomUUID } from "node:crypto"; @@ -548,6 +549,21 @@ describe("PaperclipControlPlanePort conformance", () => { payload: { itemId: "ending-item", kind: "commandExecution", text: "***REDACTED***" } }]); }); + it("records the lateness of each committed runner event once for the server metrics", async () => { + const metrics = startServerMetrics(); + try { + const f = await terminalObserverFixture(); + const event = { ...f.event, emittedAt: new Date(Date.now() - 2_000).toISOString() }; + await f.port.appendEvent(event); + await f.port.appendEvent(event); + const snapshot = metrics.snapshot(); + expect(snapshot.runEventLatenessSamples).toBe(1); + expect(snapshot.runEventLatenessP99Ms).toBeGreaterThanOrEqual(1_900); + } finally { + stopServerMetrics(metrics); + } + }); + it("completes a native run when a bound secret equals a protocol word", async () => { const observed: PrpEvent[] = []; const f = await terminalObserverFixture({ secrets: ["interrupted", "completed", "succeeded", "satisfied"], @@ -1020,6 +1036,27 @@ describe("PaperclipControlPlanePort conformance", () => { return { port, binding, identity, runnerId, event, stored, streamed }; }; + it("measures every row of a streaming batch from the batch commit, not after earlier rows' observers", async () => { + const metrics = startServerMetrics(); + const recorded: number[] = []; + const record = metrics.recordRunEventLatenessMs.bind(metrics); + metrics.recordRunEventLatenessMs = (ms: number) => { recorded.push(ms); record(ms); }; + try { + const f = await streamingFixture({ streamingBatchMs: 600_000, onCommittedEvent: async (event) => { + if (event.sourceSeq === 2) await new Promise(resolve => setTimeout(resolve, 1_200)); + } }); + await f.port.appendEvent(f.event(1), f.streamed); + const deltas = [2, 3, 4].map(sourceSeq => f.port.appendEvent(f.event(sourceSeq), f.streamed)); + await f.port.appendEvent(f.event(5, "tool.execution.started")); + await Promise.all(deltas); + const batch = recorded.slice(1, 4); + expect(batch).toHaveLength(3); + expect(Math.max(...batch) - Math.min(...batch)).toBeLessThan(500); + } finally { + stopServerMetrics(metrics); + } + }); + it("commits the first text delta at once and later deltas in one batch before the next other event", async () => { const committed: number[] = []; // A timer this long would time the test out: only the first delta and the diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.ts b/server/src/services/native-runtime/paperclip-control-plane-port.ts index 7601de7f3e6..7afee85af21 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.ts @@ -39,6 +39,7 @@ import { eligibleNativeTaskSessionPublication, publishNativeTaskSessionInTransac import { lockNativeRunnerCheckpointOwner, nativeRunnerCheckpointOwnerLock, validateNativeRunnerCheckpointOwner, inspectNativeRunnerUpgradeHeader, type NativeRunnerCheckpointOwner } from "./native-runner-checkpoint.js"; import { crashPoint } from "@paperclipai/shared/crash-points"; import { NATIVE_PROVIDER_DURABLE_EVENT_TYPES } from "./native-retired-backup-source.js"; +import { recordRunEventCommitted } from "../server-metrics.js"; export interface PaperclipControlPlaneBinding { companyId: string; @@ -476,10 +477,11 @@ export class PaperclipControlPlanePort implements ControlPlanePort { this.#assertActive(options); const pendingObservation = this.#deferredEventObservers.get(observerKey); const persisted = await appendHeartbeatRunEvent(this.#db, input); + const committedAtMs = Date.now(); this.#assertActive(options); const persistenceEndedAt = persistenceStartedAt + performance.now() - persistenceClock; let observation = pendingObservation ?? this.#deferredEventObservers.get(observerKey); - if (!observation) observation = this.#observePersistedEvent(event, persisted, persistenceStartedAt, persistenceEndedAt); + if (!observation) observation = this.#observePersistedEvent(event, persisted, persistenceStartedAt, persistenceEndedAt, committedAtMs); if (options?.deferPostCommitObservers) { this.#deferredEventObservers.set(observerKey, observation); void observation.then(() => { @@ -503,9 +505,11 @@ export class PaperclipControlPlanePort implements ControlPlanePort { persisted: AppendHeartbeatRunEventResult, persistenceStartedAt: number, persistenceEndedAt: number, + committedAtMs: number, ): Promise { try { await this.#onEventPersistence?.(event, persistenceStartedAt, persistenceEndedAt); } catch {} if (persisted.disposition === "committed") { + recordRunEventCommitted(event.emittedAt, committedAtMs); try { publishLiveEvent({ companyId: this.#binding.companyId, @@ -590,11 +594,12 @@ export class PaperclipControlPlanePort implements ControlPlanePort { for (const pending of batch) pending.reject(error); return; } + const committedAtMs = Date.now(); for (const [index, pending] of batch.entries()) { const persisted = receipts[index]!; const endedAt = pending.startedAtMs + performance.now() - pending.startedClock; try { - await this.#observePersistedEvent(pending.event, persisted, pending.startedAtMs, endedAt); + await this.#observePersistedEvent(pending.event, persisted, pending.startedAtMs, endedAt, committedAtMs); } catch (error) { pending.reject(error); continue; diff --git a/server/src/services/server-metrics.ts b/server/src/services/server-metrics.ts new file mode 100644 index 00000000000..a4ee23616d3 --- /dev/null +++ b/server/src/services/server-metrics.ts @@ -0,0 +1,274 @@ +import { randomBytes } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { + createHistogram, + monitorEventLoopDelay, + performance, + type EventLoopUtilization, + type IntervalHistogram, + type RecordableHistogram, +} from "node:perf_hooks"; +import type { RequestHandler } from "express"; +import { databaseStatementCount, type DedicatedDbConnectionStats } from "@paperclipai/db"; +import { readBuildCommit } from "../build-commit.js"; + +export const SERVER_METRICS_ENV = "PAPERCLIP_SERVER_METRICS"; +export const SERVER_METRICS_SCHEMA = "paperclip.server-metrics.v1"; +export const SERVER_METRICS_BUCKET_MS = 10_000; +export const SERVER_METRICS_BUCKETS = 6; +const EVENT_LOOP_DELAY_RESOLUTION_MS = 20; +const LINUX_CLOCK_TICK_MS = 10; + +export type ServerMetricsSnapshot = { + schema: typeof SERVER_METRICS_SCHEMA; + processId: string; + buildRevision: string | null; + uptimeSec: number; + sampledAt: string; + windowMs: number; + mainThreadCpu: number | null; + processCpu: number; + eventLoopUtilization: number; + eventLoopDelayP99Ms: number; + statementsPerSecond: number; + runEventLatenessP99Ms: number | null; + runEventLatenessSamples: number; + agentApiPreHandlerP99Ms: number | null; + agentApiPreHandlerSamples: number; + leaseRenewalsMissed: number; + dedicatedConnections: DedicatedDbConnectionStats | null; +}; + +export type ServerMetricsProbes = { + now: () => number; + wallNow: () => number; + eventLoopUtilization: (current?: EventLoopUtilization, previous?: EventLoopUtilization) => EventLoopUtilization; + processCpuMicros: () => number; + mainThreadCpuMs: () => number | null; + statementCount: () => number; + uptimeSec: () => number; + dedicatedConnections: () => DedicatedDbConnectionStats | null; +}; + +type Checkpoint = { + at: number; + elu: EventLoopUtilization; + processCpuMicros: number; + mainThreadCpuMs: number | null; + statements: number; +}; + +function readLinuxMainThreadCpuMs(): number | null { + if (process.platform !== "linux") return null; + try { + const stat = readFileSync(`/proc/self/task/${process.pid}/stat`, "utf8"); + const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" "); + const ticks = Number(fields[11]) + Number(fields[12]); + return Number.isFinite(ticks) ? ticks * LINUX_CLOCK_TICK_MS : null; + } catch { + return null; + } +} + +const defaultProbes: ServerMetricsProbes = { + now: () => performance.now(), + wallNow: () => Date.now(), + eventLoopUtilization: (current, previous) => performance.eventLoopUtilization(current, previous), + processCpuMicros: () => { + const usage = process.cpuUsage(); + return usage.user + usage.system; + }, + mainThreadCpuMs: readLinuxMainThreadCpuMs, + statementCount: databaseStatementCount, + uptimeSec: () => process.uptime(), + dedicatedConnections: () => null, +}; + +const round = (value: number, digits: number) => Number(value.toFixed(digits)); + +export class ServerMetricsCollector { + readonly processId: string; + readonly buildRevision: string | null; + private readonly probes: ServerMetricsProbes; + private readonly bucketMs: number; + private readonly starts: Array; + private readonly lateness: RecordableHistogram[]; + private readonly preHandler: RecordableHistogram[]; + private readonly delayP99Ns: number[]; + private readonly merged = createHistogram(); + private readonly eventLoopDelay: IntervalHistogram | null; + private current = 0; + private missedLeaseRenewals = 0; + private timer: NodeJS.Timeout | null = null; + private stopped = false; + + constructor(options: { probes?: Partial; bucketMs?: number; buckets?: number; + buildRevision?: string | null; monitorEventLoop?: boolean; processId?: string } = {}) { + this.processId = options.processId ?? randomBytes(8).toString("hex"); + this.probes = { ...defaultProbes, ...options.probes }; + this.bucketMs = options.bucketMs ?? SERVER_METRICS_BUCKET_MS; + const buckets = options.buckets ?? SERVER_METRICS_BUCKETS; + this.buildRevision = options.buildRevision === undefined ? readBuildCommit() : options.buildRevision; + this.starts = Array.from({ length: buckets }, () => null); + this.lateness = Array.from({ length: buckets }, () => createHistogram()); + this.preHandler = Array.from({ length: buckets }, () => createHistogram()); + this.delayP99Ns = Array.from({ length: buckets }, () => 0); + this.eventLoopDelay = options.monitorEventLoop === false ? null + : monitorEventLoopDelay({ resolution: EVENT_LOOP_DELAY_RESOLUTION_MS }); + this.eventLoopDelay?.enable(); + this.starts[0] = this.checkpoint(); + this.arm(); + } + + private checkpoint(): Checkpoint { + return { + at: this.probes.now(), + elu: this.probes.eventLoopUtilization(), + processCpuMicros: this.probes.processCpuMicros(), + mainThreadCpuMs: this.probes.mainThreadCpuMs(), + statements: this.probes.statementCount(), + }; + } + + private arm(): void { + if (this.stopped) return; + this.timer = setTimeout(() => { + this.timer = null; + this.rotate(); + this.arm(); + }, this.bucketMs); + this.timer.unref(); + } + + rotate(): void { + const now = this.checkpoint(); + const started = this.starts[this.current]!; + const steps = Math.min(this.starts.length - 1, Math.max(1, Math.floor((now.at - started.at) / this.bucketMs))); + if (this.eventLoopDelay) { + this.delayP99Ns[this.current] = this.eventLoopDelay.percentile(99); + this.eventLoopDelay.reset(); + } + for (let step = 0; step < steps; step += 1) { + this.current = (this.current + 1) % this.starts.length; + this.lateness[this.current]!.reset(); + this.preHandler[this.current]!.reset(); + this.delayP99Ns[this.current] = 0; + this.starts[this.current] = null; + } + this.starts[this.current] = now; + } + + recordRunEventLatenessMs(ms: number): void { + this.lateness[this.current]!.record(Math.max(1, Math.round(ms * 1000))); + } + + recordAgentApiPreHandlerMs(ms: number): void { + this.preHandler[this.current]!.record(Math.max(1, Math.round(ms * 1000))); + } + + recordMissedLeaseRenewal(): void { + this.missedLeaseRenewals += 1; + } + + private windowP99Ms(histograms: RecordableHistogram[]): { p99Ms: number | null; samples: number } { + this.merged.reset(); + for (const histogram of histograms) if (histogram.count > 0) this.merged.add(histogram); + const samples = this.merged.count; + return { p99Ms: samples > 0 ? round(this.merged.percentile(99) / 1000, 3) : null, samples }; + } + + snapshot(): ServerMetricsSnapshot { + const now = this.checkpoint(); + let oldest = this.starts[this.current]!; + for (let offset = 1; offset < this.starts.length; offset += 1) { + const start = this.starts[(this.current + offset) % this.starts.length]; + if (start) { oldest = start; break; } + } + const windowMs = Math.max(1, now.at - oldest.at); + let delayP99Ns = this.eventLoopDelay?.percentile(99) ?? 0; + for (let index = 0; index < this.starts.length; index += 1) { + if (index !== this.current && this.starts[index]) delayP99Ns = Math.max(delayP99Ns, this.delayP99Ns[index]!); + } + const lateness = this.windowP99Ms(this.lateness); + const preHandler = this.windowP99Ms(this.preHandler); + return { + schema: SERVER_METRICS_SCHEMA, + processId: this.processId, + buildRevision: this.buildRevision, + uptimeSec: Math.round(this.probes.uptimeSec()), + sampledAt: new Date(this.probes.wallNow()).toISOString(), + windowMs: Math.round(windowMs), + mainThreadCpu: now.mainThreadCpuMs !== null && oldest.mainThreadCpuMs !== null + ? round((now.mainThreadCpuMs - oldest.mainThreadCpuMs) / windowMs, 4) : null, + processCpu: round((now.processCpuMicros - oldest.processCpuMicros) / 1000 / windowMs, 4), + eventLoopUtilization: round(this.probes.eventLoopUtilization(now.elu, oldest.elu).utilization, 4), + eventLoopDelayP99Ms: round(delayP99Ns / 1e6, 3), + statementsPerSecond: round((now.statements - oldest.statements) / (windowMs / 1000), 2), + runEventLatenessP99Ms: lateness.p99Ms, + runEventLatenessSamples: lateness.samples, + agentApiPreHandlerP99Ms: preHandler.p99Ms, + agentApiPreHandlerSamples: preHandler.samples, + leaseRenewalsMissed: this.missedLeaseRenewals, + dedicatedConnections: this.probes.dedicatedConnections(), + }; + } + + stop(): void { + this.stopped = true; + if (this.timer) clearTimeout(this.timer); + this.timer = null; + this.eventLoopDelay?.disable(); + } +} + +let active: ServerMetricsCollector | null = null; + +export function serverMetricsEnabledFromEnv(env: NodeJS.ProcessEnv): boolean { + const value = env[SERVER_METRICS_ENV]?.trim().toLowerCase(); + if (value === undefined || value === "") return false; + if (value === "true" || value === "1") return true; + if (value === "false" || value === "0") return false; + throw new Error(`${SERVER_METRICS_ENV} must be "true" or "false", got: ${env[SERVER_METRICS_ENV]}`); +} + +export function startServerMetrics(dedicatedConnections?: () => DedicatedDbConnectionStats | null): ServerMetricsCollector { + active ??= new ServerMetricsCollector(dedicatedConnections ? { probes: { dedicatedConnections } } : {}); + return active; +} + +export function stopServerMetrics(collector: ServerMetricsCollector): void { + collector.stop(); + if (active === collector) active = null; +} + +export function activeServerMetrics(): ServerMetricsCollector | null { + return active; +} + +export function recordRunEventCommitted(emittedAt: string | null | undefined, committedAtMs: number): void { + if (!active || !emittedAt) return; + const emitted = Date.parse(emittedAt); + if (Number.isFinite(emitted)) active.recordRunEventLatenessMs(committedAtMs - emitted); +} + +export function recordMissedLeaseRenewal(): void { + active?.recordMissedLeaseRenewal(); +} + +const arrivedAt = Symbol("serverMetricsArrivedAt"); +type TimedRequest = { [arrivedAt]?: number }; + +export function serverMetricsArrival(): RequestHandler { + return (req, _res, next) => { + (req as TimedRequest)[arrivedAt] = performance.now(); + next(); + }; +} + +export function serverMetricsAgentPreHandler(collector: ServerMetricsCollector): RequestHandler { + return (req, _res, next) => { + const started = (req as TimedRequest)[arrivedAt]; + if (started !== undefined && req.actor?.type === "agent") collector.recordAgentApiPreHandlerMs(performance.now() - started); + next(); + }; +}