diff --git a/doc/plans/2026-11-13-thousand-concurrent-agents.md b/doc/plans/2026-11-13-thousand-concurrent-agents.md new file mode 100644 index 00000000000..c43512d9657 --- /dev/null +++ b/doc/plans/2026-11-13-thousand-concurrent-agents.md @@ -0,0 +1,219 @@ +# Plan: 1,000 concurrent agents on one workspace + +Date: 2026-11-13. Status: Phase 1 changes 1a to 1e and 1h are merged; 1f, 1g +and Phase 2 are not started. + +## Finish line + +One hosted workspace runs 1,000 agents at the same time, served by several +server processes, and all of these hold for one hour: + +| Measure | Target | +| --- | --- | +| Runs that need Retry | 0 | +| Main-thread CPU of every server process | below 60% of one core | +| Run event lateness (runner emits → row committed), p99 | below 1 s | +| Agent API request time before the handler, p99 | below 100 ms | +| Lease renewals that miss their deadline | 0 | +| Board page API requests, p90 | below 500 ms | +| Killing one server process | its runs resume on another, no Retry | + +Intermediate gates: 50 agents on one process, then 200 on one process, then +1,000 on several. + +## Why 50 struggles today + +One Node process controls every run of a workspace. Its main thread is the +limit, not the database. + +| Evidence | Result | Source | +| --- | --- | --- | +| Main thread of the staging server | 93–96% of one core, container limit 24 | OS thread sample, Nov 11 and 12 | +| Database CPU | 1–2 vCPU of about 8; most "active" sessions wait for the app | Neon, Nov 11 | +| Building SQL text in Node | 24% of main-thread CPU, 46% of 78 MB/s allocated | CPU profile and allocation sample, Nov 11–12 | +| Garbage collection | 13–25% of the main thread | same profiles | + +The per-agent work that produces that load, read from the code: + +| Source | When | Server work | +| --- | --- | --- | +| Run events | every event, also every text delta | one transaction of 4 round trips, a run-row lock, a count over the run's events, 2 hashes, 3 or more JSON serialisations (`heartbeat-run-events.ts:172`, `paperclip-control-plane-port.ts:360`) | +| Controller checkpoint | every event that is not a delta, every command | serialise the journal, encrypt, one transaction on a 3-connection pool (`durable-prp-control-plane.ts:403`, `native-runner-checkpoint.ts:128`) | +| Agent API and tool calls | every request | about 13 round trips before the handler, none cached (`middleware/auth.ts:326`, `run-identity.ts:546`) | +| Lease renewal | every 20 s per remote run | one transaction on a 2-connection pool; wakes 6 recovery queues (`native-execution-lease.ts:311`) | +| Run log to object storage | 1 s timer per run | 4 upload slots for the whole process (`object-run-log-store.ts:72`) | +| Agent-file checkpoint | end of every turn | agent row lock held across file walk and upload; 2-connection pool (`agent-file-store.ts:197`) | +| Workspace snapshot, harness backup | run end, session close | archive bytes pass through the server | + +None of the event work depends on a viewer. Only the last hop to a browser +socket is gated on a subscription (`realtime/live-event-delivery.ts`). + +## Load model result + +`scripts/benchmark-agent-scaling-load.ts` drives the real append, checkpoint, +lease-renewal and agent-authentication code for N simulated agents in one +process against a local PostgreSQL. Assumed per-agent rates: 20 deltas/s, +1 other event/s, 0.3 API requests/s, one renewal per 20 s. 20 s window, main +pool of 10 connections unless stated. Measured 2026-11-13 on master before any +Phase 1 fix, machine load average 11–17. + +| Agents | Scenario | Statements/s | Event-loop utilization | Deltas done of offered | Delta lateness p50 / p99 | API auth p50 / p99 | +| --- | --- | --- | --- | --- | --- | --- | +| 50 | All sources | 4,730 | 0.37–0.50 | 20,000 of 20,000 | 0.8 ms / 4–63 ms | 4–5 ms / 8–196 ms | +| 50 | All sources, deltas at 2/s | 1,130 | 0.16 | 2,000 of 2,000 | 1.2 ms / 5 ms | 5 ms / 24 ms | +| 50 | Everything except deltas | 731 | 0.13 | — | — | 5 ms / 13 ms | +| 50 | Deltas only | 4,000 | 0.39 | 20,000 of 20,000 | 0.9 ms / 22 ms | — | +| 100 | All sources | 9,460 | 0.72 | 40,000 of 40,000 | 1.2 ms / 24 ms | 6 ms / 93 ms | +| 150 | All sources | 12,300 | 0.83 | 50,788 of 60,000 | 1.1 s / 3.1 s | 268 ms / 422 ms | +| 200 | All sources | 12,180 | 0.92 | 50,976 of 80,000 | 3.5 s / 6.4 s | 352 ms / 2.3 s | +| 200 | All sources, pool of 30 | 12,670 | 0.91 | 49,173 of 80,000 | 4.2 s / 7.6 s | 350 ms / 597 ms | +| 200 | All sources, deltas at 2/s | 4,520 | 0.51 | 8,000 of 8,000 | 1.0 ms / 29 ms | 5 ms / 42 ms | + +What it shows: + +- Deltas are about 85% of the statements (4,000/s of 4,730/s at 50 agents). +- The process keeps up at 50 and 100 agents and saturates between 100 and 150, + at about 12,300 statements/s. Past that point deltas fall seconds behind and + agent authentication slows from 5 ms to hundreds of milliseconds. +- The limit is the Node main thread, not the pool: event-loop utilization is + 0.83–0.92 at saturation and a pool of 30 changes nothing. +- These sources alone use 37–50% of the event loop at 50 agents, with no + network latency. A hosted server also runs tool calls, board requests and + recovery sweeps on the same thread, which is consistent with staging + saturating near 50. +- Ten times fewer delta transactions serve 200 agents at an event-loop + utilization of 0.51. This is the case for change 1a. +- Lease renewal is negligible at these sizes (15 statements/s at 50 agents). + +Limits: the machine carried other work, so timings vary between runs (the +ranges in the 50-agent row are two runs); statement counts do not. A first +pass under a load average of 40–60 finished only half the deltas at 50 agents; +that was the machine, not the server code, and it is not shown. Raw output is +not kept in the repository. + +## Load model after Phase 1 + +Same model and rates, measured 2026-11-13 with the eight Phase 1 fixes below +(1a to 1e and 1h). The model now commits deltas as the server's port does (the +first text after another event at once, later deltas in one transaction per +250 ms or 100 deltas) and renews leases through the shared schedule. Machine +load average 11–15. + +| Agents | Statements/s | Event-loop utilization | Deltas done of offered | Delta lateness p50 / p99 | Other events done of offered | API auth p50 / p99 | +| --- | --- | --- | --- | --- | --- | --- | +| 50 | 1,780 | 0.26 | 20,000 of 20,000 | 103 ms / 253 ms | 1,000 of 1,000 | 2 ms / 10 ms | +| 100 | 3,570 | 0.46 | 40,000 of 40,000 | 105 ms / 258 ms | 2,000 of 2,000 | 2 ms / 69 ms | +| 200 | 7,110 | 0.73 | 80,000 of 80,000 | 151 ms / 573 ms | 4,000 of 4,000 | 3 ms / 452 ms | +| 400 | 11,380 | 0.99 | not reliable for this run (1) | 1.7 s / 2.2 s | 5,473 of 8,000 | 394 ms / 620 ms | +| 200, deltas and leases as before Phase 1 | 10,220 | 0.90 | 39,425 of 80,000 | 5.1 s / 10.2 s | 4,000 of 4,000 | 165 ms / 1.8 s | + +(1) The model that measured this table still counted a delta that committed +after the window, while queued checkpoints drained. At 400 agents deltas ran +1.7 s late, so its count includes deltas outside the window. The model now +counts only deltas that commit inside the window; this row was not measured +again. At 50 to 200 agents every delta committed within its batch time, so +those counts are unaffected. + +What it shows: + +- Statements per agent per second fall from about 95 to about 36. +- One process now serves 200 agents with every delta committed within its + batch time, where before it finished half of them and ran seconds behind. It + saturates between 200 and 400 agents, against 100 to 150 before. +- A batched delta is committed up to 250 ms after it arrives. That is the + design, not a backlog; the first text after another event does not wait. +- The ceiling is still about 12,000 statements/s on one thread. Delta batches + are still the largest share: 4 transactions of 5 statements per agent per + second. A longer batch time, or one transaction for the deltas of many runs, + is the next step on one process; Phase 2 is the step past it. +- Lease renewal is under 1 statement/s at 400 agents (15/s at 50 before). +- The model does not measure server memory, so it does not show change 1h. + +## Why more servers do not help today + +| Assumption | Where | Effect with two steady processes | +| --- | --- | --- | +| Only the newest process starts runs | `run-start-handover.ts:10`, `deployment-notifications.ts:11` | the older one takes no new work | +| A run's socket, timers and session state live in one process | `runner-prp-outbound.ts:242`, `native-session-executor.ts:416` | Stop, steer and answers must reach the owner | +| Live events are an in-process emitter | `live-events.ts:7` | a browser on the other process sees nothing | +| The per-agent start lock is an in-process map | `agent-start-lock.ts:4` | two processes can exceed an agent's run limit | +| Cross-process wakes carry no scope | `packages/db/src/work-signals.ts:79` | each renewal makes every other process rescan every run | +| Every pool and upload limiter is per process | `packages/db/src/client.ts:306,375,454` | limits multiply, nothing is coordinated | + +The fencing model for a second process exists (boot id, epoch, leases in +`controller-instance-lease.ts`), but it was built for deploy overlap. + +## Phases + +### Phase 0: measure (this week) + +1. Land the load model and record its baseline (done locally, see above). +2. Add a per-run statement and transaction counter to the server's existing + database response capture, exported as a metric, so hosted runs report + "statements per agent per second" by source. +3. Build a scripted runner that replays a recorded event stream at a set rate + with no model call, so 50 to 1,000 real runs cost only sandbox time. +4. Run 50 scripted agents on a hosted staging stack that is not + `paperclip.staging`, with a CPU profile. This confirms or corrects the + ranking on hosted round-trip times. + +### Phase 1: cut the per-agent cost (target: 200 agents on one process) + +Each item is one PR with a red/green test and a before/after from the load +model. Item 1h is about memory, which the load model does not measure; its +before/after is retained heap per session. The catalog decides which tool +schemas a session may call, so the cache key must cover everything that +decides the admitted schemas, and nothing may be shared across companies. +Warm sessions have no cap in number and stay five minutes after the last +turn, so memory per session counts for idle agents too. + +| # | Change | Expected effect | +| --- | --- | --- | +| 1a | Commit deltas in batches: one transaction per runner frame batch or per 250 ms per run, acknowledging the runner after the batch commits | delta transactions per agent from about 20/s to at most 4/s | +| 1b | Keep the retained count and highest sequence on the run row instead of counting event rows on every append | append cost stops growing with run length | +| 1c | Read the run's identity once per identity version and reuse it for the run's requests, keeping the per-request revocation read | pre-handler round trips from about 13 to 3 | +| 1d | Renew all of a process's leases in one statement per tick, as the legacy lease already does | renewals stop scaling with run count; no 2-connection queue | +| 1e | Serialise only the changed journal parts for a checkpoint; skip the agent-file transaction when the tree is unchanged | checkpoint CPU stops growing with journal length | +| 1f | Upload archives from the sandbox straight to object storage with a signed URL; the server records the digest | no archive bytes on the server | +| 1g | Size the shared limiters (log uploads, checkpoint pool, recovery pass) from the active run count | no fixed slot count below the run count | +| 1h | Keep one frozen, parsed tool catalog per catalog hash and company, shared by the sessions that use it, in place of two private copies per session (`assigned-mcp-tools.ts:146`) | server memory per live or warm session from about 5–11 MB (measured at 400–800 tools) to near zero; at 1,000 agents that is 5–11 GB | + +### Phase 2: several controller processes (target: 1,000) + +| # | Change | +| --- | --- | +| 2a | Any live process may claim a queued run. Replace "newest process starts everything" with a per-run claim; keep epoch fencing for deploys | +| 2b | Route Stop, steer, answers and approvals to the owning process through a durable per-run command row plus a wake | +| 2c | Publish live events across processes (PostgreSQL notify with the event id, body read from the row), so a browser can connect anywhere | +| 2d | Move the per-agent start lock and concurrency count into the database claim | +| 2e | Give cross-process wakes a scope, and let each process sweep only the runs it owns; a dead owner's runs are found by lease expiry | +| 2f | Separate the roles: API processes serve the board and agent requests, controller processes own runs; scale each on its own | + +2a and 2e change run ownership, so they must pass the restart-chaos waves at +0% Retry before any hosted rollout. + +### Phase 3: prove it + +1. Ladder with the scripted runner: 50, 200, 500, 1,000 agents, one hour + each, on a hosted stack that is not `paperclip.staging`. +2. Kill a controller process at each step and check the finish-line table. +3. `paperclip.staging` only after the owner's explicit go-ahead. +4. Repeat the 1,000 step with real model calls for a short window. + +## Decisions for the owner + +| # | Decision | Recommendation | +| --- | --- | --- | +| 1 | May deltas be stored as one row per batch instead of one row per delta? Replay and the transcript stay complete; a crash can lose at most the unacknowledged batch, which the runner resends | yes; approved by the owner 2026-11-13 | +| 2 | Bus for cross-process events: PostgreSQL notify or a new Redis dependency | PostgreSQL first; revisit above 1,000 | +| 3 | Order of work: Phase 1 before Phase 2 | yes; at today's cost one process saturates between 100 and 150 agents in the load model, and near 50 on a hosted server | + +## Not verified + +- The load model runs on loopback PostgreSQL on a developer machine. It + measures the modelled sources only and has no hosted round-trip time. +- The per-agent rates (20 deltas/s, 1 other event/s, 0.3 API requests/s) are + assumptions, not a recorded workload. Phase 0 step 2 replaces them. +- Staging's pool size and run-log storage mode were not read. +- Tool-gateway calls, recovery sweeps, and browser fan-out are not in the + model. diff --git a/scripts/benchmark-agent-scaling-load.ts b/scripts/benchmark-agent-scaling-load.ts new file mode 100644 index 00000000000..407dba26e51 --- /dev/null +++ b/scripts/benchmark-agent-scaling-load.ts @@ -0,0 +1,314 @@ +/** + * Load model: what N running native agents cost one server process. + * + * Each simulated agent drives the real server code at a fixed rate, against an + * embedded PostgreSQL on loopback, in this one Node process: + * + * deltas an `item.delta` per text chunk, committed as the server's port + * commits them: the first after another event at once, later ones + * in one transaction per DELTA_BATCH_MS or 100 deltas + * (DELTA_BATCH_MS=0: one transaction per delta, as before batching) + * events appendHeartbeatRunEvent for an `item.completed`, then the real + * controller checkpoint save, which overlaps the next append + * lease every LEASE_S, one renewal of all leases through the shared + * schedule (LEASE_MODE=single: one transaction per run, as before). + * In batch mode LEASE_S is at most 20: the schedule renews a remote + * lease on its own 20 s after its last renewal. + * api an HTTP request through actorMiddleware with the run's agent JWT + * + * Arrivals are open-loop: operation i of a stream is due at start + i/rate. + * A stream is serial, like one runner connection, so when the process cannot + * keep up the lateness (finish time minus due time) grows and the stream + * finishes fewer operations than were offered in the window. + * + * It reports, for the window: event-loop utilization and delay, CPU, memory + * allocated, statements and transactions, and per source the operations done + * against those offered with their lateness. An operation that settles after + * the window (and the bounded wait for the last delta batches) is not done. + * + * Not modelled: frame decrypt and validation, redaction, the object run log, + * live-event delivery to browsers, tool-gateway and route handler bodies, + * recovery sweeps, and network round-trip time to a hosted database. So the + * figures are a floor for the modelled sources, not a whole-server prediction. + * + * pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-agent-scaling-load.ts + * + * Optional: AGENTS (50), SECONDS (20), SOURCES (deltas,events,lease,api), + * DELTAS_PER_S (20), EVENTS_PER_S (1), API_PER_S (0.3), LEASE_S (20), + * DELTA_BYTES (120), EVENT_KIB (2), DELTA_BATCH_MS (250), LEASE_MODE (batch), + * DATABASE_POOL_MAX (driver default 10). + */ +import assert from "node:assert/strict"; +import { randomBytes, randomUUID } from "node:crypto"; +import type { AddressInfo } from "node:net"; +import { monitorEventLoopDelay, performance } from "node:perf_hooks"; +import { createRequire } from "node:module"; +import { agents, checkpointDb, companies, createDb, heartbeatRuns, issues, livenessDb, nativeRunFinalizations, + startEmbeddedPostgresTestDatabase } from "../packages/db/src/index.js"; +import { meterDatabaseStatements } from "../server/src/__tests__/helpers/database-statement-meter.js"; +import { createLocalAgentJwt } from "../server/src/agent-auth-jwt.js"; +import { actorMiddleware } from "../server/src/middleware/auth.js"; +import { appendHeartbeatRunEvent, appendHeartbeatRunEventBatch, type AppendHeartbeatRunEventInput } from "../server/src/services/heartbeat-run-events.js"; +import { renewNativeExecutionLeasesNow, renewNativeSessionExecutionLease, startNativeSessionExecutionLeaseRenewal } + from "../server/src/services/native-runtime/native-execution-lease.js"; +import { nativeRunnerCheckpointWriter } from "../server/src/services/native-runtime/native-runner-checkpoint.js"; +import { initializeRunIdentity } from "../server/src/services/run-identity.js"; + +const serverRequire = createRequire(new URL("../server/package.json", import.meta.url)); +const express = serverRequire("express") as typeof import("express"); + +const number = (name: string, fallback: number) => Number(process.env[name] ?? fallback); +const AGENTS = number("AGENTS", 50), SECONDS = number("SECONDS", 20); +const RATES = { deltas: number("DELTAS_PER_S", 20), events: number("EVENTS_PER_S", 1), api: number("API_PER_S", 0.3), + lease: 1 / number("LEASE_S", 20) }; +type Source = keyof typeof RATES; +const SOURCES = (process.env.SOURCES ?? "deltas,events,lease,api").split(",").map(value => value.trim()).filter(Boolean) as Source[]; +for (const source of SOURCES) assert(source in RATES, `Unknown source ${source}`); +const DELTA_BYTES = number("DELTA_BYTES", 120), EVENT_KIB = number("EVENT_KIB", 2); +const DELTA_BATCH_MS = number("DELTA_BATCH_MS", 250), DELTA_BATCH_MAX = 100; +const LEASE_BATCHED = (process.env.LEASE_MODE ?? "batch") !== "single"; +// The shared schedule renews a remote lease on its own 20 s after the last +// renewal. The model drives every renewal itself, so a tick must come sooner, +// or the schedule's own renewals would add work that the model does not count. +assert(!LEASE_BATCHED || !SOURCES.includes("lease") || 1 / RATES.lease <= 20, + "LEASE_MODE=batch needs LEASE_S of at most 20: the shared schedule renews on its own after 20 s"); +process.env.PAPERCLIP_SECRETS_MASTER_KEY ??= randomBytes(32).toString("base64"); +process.env.PAPERCLIP_AGENT_JWT_SECRET ??= randomBytes(32).toString("base64"); + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, Math.max(0, ms))); +const quantile = (values: number[], q: number) => values.length === 0 ? null + : Number([...values].sort((a, b) => a - b)[Math.min(values.length - 1, Math.floor(values.length * q))]!.toFixed(1)); +const round = (value: number, digits = 1) => Number(value.toFixed(digits)); + +const temporary = await startEmbeddedPostgresTestDatabase("agent-scaling-load-"); +const db = createDb(temporary.connectionString); +// Checkpoint saves and lease renewals use their own pools; meter all three. +const POOLS = ["main", "checkpoint", "lease"] as const; +const meters = [db, checkpointDb(db), livenessDb(db)].map(client => meterDatabaseStatements(client, temporary.connectionString)); +const poolStatements = () => meters.map(one => one.read().statements); +const meter = { read: () => meters.map(one => one.read()).reduce((sum, one) => ({ statements: sum.statements + one.statements, + transactions: sum.transactions + one.transactions, sqlBytes: sum.sqlBytes + one.sqlBytes }), { statements: 0, transactions: 0, sqlBytes: 0 }) }; +const controller = { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() }; +let server: import("node:http").Server | undefined; +try { + const [company] = await db.insert(companies).values({ name: "Scaling load", issuePrefix: "LOAD" }).returning(); + const companyId = company!.id; + const fleet = []; + for (let index = 0; index < AGENTS; index++) { + const [agent] = await db.insert(agents).values({ companyId, name: `Agent ${index}`, adapterType: "process" }).returning(); + const [issue] = await db.insert(issues).values({ companyId, title: `Task ${index}` }).returning(); + const runId = randomUUID(), leaseOwner = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId: agent!.id, status: "running", runtimeMode: "native", + nativeIssueId: issue!.id, issueId: issue!.id, runnerProfileJson: { padding: randomBytes(3 * 1024).toString("base64") } }); + await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId: issue!.id, phase: "observed", attempt: 0, + controllerBootId: controller.bootId, controllerPid: controller.pid, controllerProcessStartedAt: controller.processStartedAt, + controllerGeneration: 1, leaseOwner, leaseExpiresAt: new Date(Date.now() + 3_600_000) }); + if (SOURCES.includes("api")) { + await initializeRunIdentity(db, { companyId, runId, responsibleUserId: null, issueId: issue!.id, cause: "benchmark" }); + } + const write = nativeRunnerCheckpointWriter(db, { companyId, issueId: issue!.id, runId, authorityRunId: runId, + bootId: controller.bootId, generation: 1, leaseOwner }); + const records = new Map([["header", JSON.stringify({ padding: randomBytes(6 * 1024).toString("base64") })], + ["cursor", JSON.stringify({ ackedSourceSeq: 0 })]]); + const checkpoint = (changed: Map, reset: boolean) => ({ + runId, records: new Map(records), changedRecords: changed, removedRecords: [], reset, workCommandIds: [], predecessor: null, + runnerVersion: "bench", runnerDigest: `sha256:${"a".repeat(64)}`, + recoveryEvidence: { runnerInstanceId: "bench", normalizedSessionId: "bench", authenticatedConnectionCount: 1, highestRecordedSourceSeq: 0 }, + restoreSessionDirectory() {}, + }) as unknown as Parameters[0]; + await write(checkpoint(new Map(records), true)); + fleet.push({ agentId: agent!.id, issueId: issue!.id, runId, leaseOwner, write, records, checkpoint, seq: 0, + checkpointTail: Promise.resolve(), appendTail: Promise.resolve() as Promise, + deltaQueue: [] as Array<{ input: AppendHeartbeatRunEventInput; settle: (error?: unknown) => void }>, + deltaTimer: null as ReturnType | null, streamedSinceBoundary: false, token: createLocalAgentJwt(agent!.id, companyId, "process", runId, null)! }); + } + + const app = express(); + app.use(actorMiddleware(db, { deploymentMode: "authenticated", resolveSession: async () => null })); + app.get("/actor", (req, res) => { res.json({ type: (req as unknown as { actor: { type: string } }).actor.type }); }); + server = app.listen(0, "127.0.0.1"); + await new Promise(resolve => server!.once("listening", resolve)); + const origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + + type Agent = (typeof fleet)[number]; + let checkpointFailures = 0, firstCheckpointError = ""; + // One runner connection delivers a run's events in order, and the server + // awaits each append before it reads the next. So deltas and other events + // of one agent share one queue and never append at the same time. + const eventInput = (agent: Agent, eventType: string, bytes: number) => { + const seq = ++agent.seq; + const event = { sourceSeq: seq, eventType, runId: agent.runId, text: randomBytes(Math.ceil(bytes * 0.75)).toString("base64") }; + const input: AppendHeartbeatRunEventInput = { companyId, agentId: agent.agentId, runId: agent.runId, eventType, stream: "system", + level: "info", payload: { prpEvent: event }, assertWritable: () => {}, + nativeSource: { sourceInstanceId: "bench-runner", sourceEventId: `event-${seq}`, sourceSeq: seq, protocolSchemaVersion: 1, + canonicalPayload: event } }; + return { seq, event, input }; + }; + // The queued deltas of an agent commit in one transaction, in order, before + // anything appended after them. + const flushDeltas = (agent: Agent) => { + if (agent.deltaTimer !== null) clearTimeout(agent.deltaTimer); + agent.deltaTimer = null; + const batch = agent.deltaQueue; + if (batch.length === 0) return; + agent.deltaQueue = []; + agent.streamedSinceBoundary = true; + agent.appendTail = agent.appendTail.then(() => appendHeartbeatRunEventBatch(db, batch.map(item => item.input), () => {})) + .then(() => { for (const item of batch) item.settle(); }, error => { for (const item of batch) item.settle(error); }); + }; + const appendDelta = (agent: Agent) => { + const { input } = eventInput(agent, "item.delta", DELTA_BYTES); + if (DELTA_BATCH_MS <= 0) { + const queued = agent.appendTail.then(() => appendHeartbeatRunEvent(db, input)); + agent.appendTail = queued.catch(() => undefined); + return queued; + } + return new Promise((resolve, reject) => { + agent.deltaQueue.push({ input, settle: error => error === undefined ? resolve() : reject(error) }); + // The first text after another event never waits for the timer. + if (!agent.streamedSinceBoundary || agent.deltaQueue.length >= DELTA_BATCH_MAX) flushDeltas(agent); + else agent.deltaTimer ??= setTimeout(() => flushDeltas(agent), DELTA_BATCH_MS); + }); + }; + const append = (agent: Agent, eventType: string, bytes: number) => { + const { seq, event, input } = eventInput(agent, eventType, bytes); + flushDeltas(agent); + agent.streamedSinceBoundary = false; + const queued = agent.appendTail.then(() => appendHeartbeatRunEvent(db, input)).then(() => ({ seq, event })); + agent.appendTail = queued.catch(() => undefined); + return queued; + }; + const operations: Record Promise> = { + deltas: agent => appendDelta(agent), + events: async agent => { + const { seq, event } = await append(agent, "item.completed", EVENT_KIB * 1024); + const changed = new Map([[`event:${seq}`, JSON.stringify(event)], ["cursor", JSON.stringify({ ackedSourceSeq: seq })]]); + for (const [key, value] of changed) agent.records.set(key, value); + // The save is serial per controller and does not block the next frame. + agent.checkpointTail = agent.checkpointTail.then(() => agent.write(agent.checkpoint(changed, false))).catch(error => { + checkpointFailures += 1; + firstCheckpointError ||= String((error as Error)?.message ?? error).slice(0, 200); + }); + }, + lease: agent => renewNativeSessionExecutionLease({ db, runId: agent.runId, companyId, issueId: agent.issueId, + leaseOwner: agent.leaseOwner, attempt: 0, controller, leaseTtlMs: 60_000 }), + api: async agent => { + const response = await fetch(`${origin}/actor`, { headers: { authorization: `Bearer ${agent.token}`, "x-paperclip-run-id": agent.runId } }); + const body = await response.json() as { type?: string }; + assert.equal(body.type, "agent", `agent request was not authenticated (${response.status})`); + }, + }; + + // One warm pass per source, outside the window. + for (const source of SOURCES) if (!(LEASE_BATCHED && source === "lease")) await operations[source](fleet[0]!); + await fleet[0]!.checkpointTail; + assert.equal(checkpointFailures, 0, `checkpoint save failed during warm-up: ${firstCheckpointError}`); + + const stats = Object.fromEntries(SOURCES.map(source => [source, { done: 0, failed: 0, late: [] as number[], firstError: "" }])) as + Record; + const delay = monitorEventLoopDelay({ resolution: 10 }); + const [commitsBefore] = await db.$client.unsafe("select xact_commit::float8 as commits from pg_stat_database where datname = current_database()"); + const poolsBefore = poolStatements(), statementsBefore = { ...meter.read() }, cpuBefore = process.cpuUsage(), loopBefore = performance.eventLoopUtilization(); + const heapBefore = process.memoryUsage().heapUsed; + let allocated = 0, lastHeap = heapBefore; + const sampler = setInterval(() => { const heap = process.memoryUsage().heapUsed; if (heap > lastHeap) allocated += heap - lastHeap; lastHeap = heap; }, 20); + delay.enable(); + const started = performance.now(), deadline = started + SECONDS * 1000; + // Set when the window closes: an operation that settles later is not done. + let windowClosed = false; + + // With the shared schedule every lease of the process renews in one tick, + // so the lease source is one stream for the fleet, not one per agent. + const pipelined = DELTA_BATCH_MS > 0; + const inFlight: Promise[] = []; + const sharedLease = LEASE_BATCHED && SOURCES.includes("lease"); + let leasesLost = 0; + const renewals = sharedLease ? fleet.map(agent => startNativeSessionExecutionLeaseRenewal({ db, runId: agent.runId, companyId, + issueId: agent.issueId, leaseOwner: agent.leaseOwner, attempt: 0, controller, remote: true, + onLost: async () => { leasesLost += 1; } })) : []; + const leaseTicks = sharedLease ? (async () => { + const interval = 1000 / RATES.lease, offset = Math.random() * interval, stat = stats.lease; + for (let index = 0; ; index++) { + const due = started + offset + index * interval; + if (due >= deadline) return; + await sleep(due - performance.now()); + try { await renewNativeExecutionLeasesNow(db); if (!windowClosed) stat.done += fleet.length; } + catch (error) { if (!windowClosed) stat.failed += fleet.length; stat.firstError ||= String((error as Error)?.message ?? error).slice(0, 200); } + if (!windowClosed) stat.late.push(performance.now() - due); + if (performance.now() > deadline) return; + } + })() : Promise.resolve(); + await Promise.all([leaseTicks, ...fleet.flatMap(agent => SOURCES.filter(source => !(sharedLease && source === "lease")).map(async source => { + const interval = 1000 / RATES[source], offset = Math.random() * interval, stat = stats[source]; + for (let index = 0; ; index++) { + const due = started + offset + index * interval; + if (due >= deadline) return; + await sleep(due - performance.now()); + const settled = operations[source](agent).then(() => { if (!windowClosed) stat.done += 1; }, + error => { if (!windowClosed) stat.failed += 1; stat.firstError ||= String((error as Error)?.message ?? error).slice(0, 200); }) + .then(() => { if (!windowClosed) stat.late.push(performance.now() - due); }); + // The runner does not wait for a batched delta's receipt before it + // sends the next one; every other operation is awaited in turn. + if (pipelined && source === "deltas") inFlight.push(settled); + else await settled; + // A stream that is still behind at the end of the window stops there: + // `done` below `offered` is the work the process could not keep up with. + if (performance.now() > deadline) return; + } + }))]); + // Deltas still queued at the end commit with their batch; count only those + // that settle within one batch time of the window's end. + for (const agent of fleet) flushDeltas(agent); + await Promise.race([Promise.all(inFlight), sleep(DELTA_BATCH_MS + 250)]); + // Freeze the per-source counts with the other measurements below. A delta + // that commits after this, while the queued work drains, is not done. + windowClosed = true; + await Promise.all(renewals.map(renewal => renewal.stop())); + for (const agent of fleet) if (agent.deltaTimer !== null) clearTimeout(agent.deltaTimer); + const elapsedMs = performance.now() - started; + delay.disable(); + clearInterval(sampler); + const loop = performance.eventLoopUtilization(loopBefore), cpu = process.cpuUsage(cpuBefore), statementsAfter = meter.read(), poolsAfter = poolStatements(); + // Read the commit counter before the checkpoint saves still queued drain, + // so it covers the same window as the statement totals. + const [commitsAfter] = await db.$client.unsafe("select xact_commit::float8 as commits from pg_stat_database where datname = current_database()"); + await Promise.all(fleet.map(agent => agent.checkpointTail)); + const seconds = elapsedMs / 1000; + + console.log(JSON.stringify({ + agents: AGENTS, windowSeconds: SECONDS, elapsedSeconds: round(seconds), sources: SOURCES, + perAgentRatesPerSecond: Object.fromEntries(SOURCES.map(source => [source, round(RATES[source], 3)])), + process: { + eventLoopUtilization: round(loop.utilization, 3), + eventLoopDelayMs: { p50: round(delay.percentile(50) / 1e6), p99: round(delay.percentile(99) / 1e6), max: round(delay.max / 1e6) }, + cpuCores: round((cpu.user + cpu.system) / 1e6 / seconds, 2), + heapGrowthSampledMBPerSecond: round(allocated / 1e6 / seconds), + }, + database: { + meteredStatementsPerSecond: round((statementsAfter.statements - statementsBefore.statements) / seconds), + meteredStatementsPerSecondByPool: Object.fromEntries(POOLS.map((pool, index) => + [pool, round((poolsAfter[index]! - poolsBefore[index]!) / seconds)])), + meteredTransactionsPerSecond: round((statementsAfter.transactions - statementsBefore.transactions) / seconds), + meteredSqlKBPerSecond: round((statementsAfter.sqlBytes - statementsBefore.sqlBytes) / 1024 / seconds), + commitsPerSecondAllPools: round((Number(commitsAfter!.commits) - Number(commitsBefore!.commits)) / seconds), + }, + bySource: Object.fromEntries(SOURCES.map(source => { + const stat = stats[source], offered = Math.floor(AGENTS * RATES[source] * SECONDS); + return [source, { offered, done: stat.done, failed: stat.failed, firstError: stat.firstError || undefined, + donePerSecond: round(stat.done / seconds), lateMs: { p50: quantile(stat.late, 0.5), p99: quantile(stat.late, 0.99), + max: stat.late.length ? round(stat.late.reduce((max, value) => Math.max(max, value), -Infinity)) : null } }]; + })), + deltaBatchMs: DELTA_BATCH_MS, leaseMode: LEASE_BATCHED ? "batch" : "single", leasesLost, + checkpointSaves: { failed: checkpointFailures, firstError: firstCheckpointError || undefined }, + }, null, 2)); +} finally { + server?.closeAllConnections(); + server?.close(); + // The side pools keep the process alive unless they are ended as well. An + // end can wait on a connection that is still closing, so it is bounded. + await Promise.race([sleep(6_000), Promise.all([db, checkpointDb(db), livenessDb(db)] + .map(client => client.$client.end({ timeout: 5 }).catch(() => undefined)))]); + await temporary.cleanup(); +} +// Metered sockets and keep-alive HTTP agents can outlive the report. +process.exit(process.exitCode ?? 0); diff --git a/server/src/__tests__/agent-scaling-load-model.test.ts b/server/src/__tests__/agent-scaling-load-model.test.ts new file mode 100644 index 00000000000..a6212c1f50a --- /dev/null +++ b/server/src/__tests__/agent-scaling-load-model.test.ts @@ -0,0 +1,44 @@ +import { execFile } from "node:child_process"; +import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { describe, expect, it } from "vitest"; + +const run = promisify(execFile); +const script = fileURLToPath(new URL("../../../scripts/benchmark-agent-scaling-load.ts", import.meta.url)); +const serverRoot = fileURLToPath(new URL("../..", import.meta.url)); +const tsx = createRequire(import.meta.url).resolve("tsx/cli"); + +describe("agent scaling load model", () => { + // The default batches deltas and renews every lease in one tick; the other + // mode is the earlier one transaction per delta and per lease. + it.each([ + { mode: "batched deltas and shared lease renewal", env: {} }, + { mode: "one transaction per delta and per lease", env: { DELTA_BATCH_MS: "0", LEASE_MODE: "single" } }, + ])("drives every source through the real server code and reports each pool ($mode)", async ({ env }) => { + const { stdout } = await run(process.execPath, [tsx, script], { + cwd: serverRoot, maxBuffer: 16 * 1024 * 1024, + env: { ...process.env, AGENTS: "2", SECONDS: "2", DELTAS_PER_S: "5", EVENTS_PER_S: "2", API_PER_S: "1", LEASE_S: "1", ...env }, + }); + const report = JSON.parse(stdout.slice(stdout.indexOf("{"))) as { + database: { meteredStatementsPerSecondByPool: Record<"main" | "checkpoint" | "lease", number> }; + checkpointSaves: { failed: number }; + bySource: Record; + }; + expect(Object.keys(report.bySource).sort()).toEqual(["api", "deltas", "events", "lease"]); + for (const [source, result] of Object.entries(report.bySource)) { + expect(result.failed, source).toBe(0); + expect(result.done, source).toBeGreaterThan(0); + // Counts freeze with the window: work that settles later is not done. + expect(result.done, source).toBeLessThanOrEqual(result.offered); + } + // A failed save is reported, not dropped. + expect(report.checkpointSaves.failed).toBe(0); + // Deltas, events and API requests use the main pool; checkpoint saves and + // lease renewals use their own. Each pool must be counted on its own, or a + // comparison understates the sources that use it. + expect(report.database.meteredStatementsPerSecondByPool.main).toBeGreaterThan(0); + expect(report.database.meteredStatementsPerSecondByPool.checkpoint).toBeGreaterThan(0); + expect(report.database.meteredStatementsPerSecondByPool.lease).toBeGreaterThan(0); + }, 240_000); +});