diff --git a/doc/plans/2026-11-13-thousand-concurrent-agents.md b/doc/plans/2026-11-13-thousand-concurrent-agents.md index 0d40f80967f..58e0164dea8 100644 --- a/doc/plans/2026-11-13-thousand-concurrent-agents.md +++ b/doc/plans/2026-11-13-thousand-concurrent-agents.md @@ -1,7 +1,7 @@ # Plan: 1,000 concurrent agents on one workspace -Date: 2026-11-13. Status: Phase 1 changes 1a to 1e and 1h are merged; 1f, 1g -and Phase 2 are not started. +Date: 2026-11-13. Status: Phase 1 changes 1a to 1e and 1h are merged; 1g is +done for run-log uploads (see "Change 1g"); 1f and Phase 2 are not started. ## Finish line @@ -211,6 +211,39 @@ What it shows: than its statement count suggests; the next changes should be judged by CPU per operation, not only by statements. +## Change 1g: shared limiters at 200 and 1,000 runs + +Run-log uploads now have one slot per 10 open run logs (at least 4, at most +21), and object journal operations have the upload slots plus 4 for reads (at +most 25, half of the S3 client's 50 sockets). Simulation with 50 ms per +object write, 0.7 log records per run per second, 20 s window: + +| Runs | Build | Concurrent writes | Log lines committed in window / offered | Commit delay p50 / p99 | +| --- | --- | --- | --- | --- | +| 200 | before (4 slots) | 4 | 1,473 / 2,669 | 8.0 s / 13.5 s | +| 200 | after | 20 | 2,646 / 2,790 | 1.1 s / 1.3 s | +| 1,000 | before (4 slots) | 4 | 1,229 / 13,499 | 43.8 s / 62.3 s | +| 1,000 | after | 21 | 7,789 / 13,667 | 7.7 s / 12.6 s | + +Not changed, and why: + +- Controller checkpoint pool (3 connections). Since #21553 a routine save is + one pipelined statement, so the pool is limited by backend execution and + commit time, not round trips. At 1 to 3 ms per save on a hosted database it + allows about 1,000–3,000 saves/s. Demand at 1,000 runs is about 380/s + (load model) to 1,400/s (recorded checkpoint rows). A larger pool needs a + per-process connection budget for the hosted database, which is an owner + decision. +- `withDedicatedDbConnection` has no bound on connections in use (it keeps at + most 4 idle). Workspace source and lifecycle locks nest two of them, so a + bound must not let outer holders block inner ones. Steady state at 1,000 + runs is about 10–20 connections; a burst of run starts after a restart can + open up to two per starting run. +- Lease renewal (2 connections, 200 rows per statement), agent-file saves (2 + connections, a short read-only check when nothing changed), run dispatch + (24 claims per agent pass, chained at once) and recovery per-pass caps do + not limit a healthy run at 1,000 runs. + ## Considered and not done - **Stop writing `run.performance.span` rows.** These spans are the largest diff --git a/server/src/lib/active-run-slots.test.ts b/server/src/lib/active-run-slots.test.ts new file mode 100644 index 00000000000..2f92f0cd7ee --- /dev/null +++ b/server/src/lib/active-run-slots.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; +import { assertActiveRunSlotBounds, slotsForActiveRuns } from "./active-run-slots.js"; + +describe("slotsForActiveRuns", () => { + const bounds = assertActiveRunSlotBounds({ runsPerSlot: 10, floor: 4, ceiling: 21 }); + + it("keeps the floor while few runs are active", () => { + expect(slotsForActiveRuns(0, bounds)).toBe(4); + expect(slotsForActiveRuns(40, bounds)).toBe(4); + }); + + it("adds a slot for each started group of active runs", () => { + expect(slotsForActiveRuns(41, bounds)).toBe(5); + expect(slotsForActiveRuns(200, bounds)).toBe(20); + }); + + it("never passes the ceiling", () => { + expect(slotsForActiveRuns(201, bounds)).toBe(21); + expect(slotsForActiveRuns(1_000, bounds)).toBe(21); + expect(slotsForActiveRuns(Number.MAX_SAFE_INTEGER, bounds)).toBe(21); + }); + + it("treats an invalid run count as no active runs", () => { + expect(slotsForActiveRuns(-5, bounds)).toBe(4); + expect(slotsForActiveRuns(Number.NaN, bounds)).toBe(4); + }); + + it("rejects bounds whose ceiling is below the floor", () => { + expect(() => assertActiveRunSlotBounds({ runsPerSlot: 10, floor: 5, ceiling: 4 })).toThrow("invalid_active_run_slot_bounds"); + expect(() => assertActiveRunSlotBounds({ runsPerSlot: 0, floor: 1, ceiling: 4 })).toThrow("invalid_active_run_slot_bounds"); + }); +}); diff --git a/server/src/lib/active-run-slots.ts b/server/src/lib/active-run-slots.ts new file mode 100644 index 00000000000..a39d636dd9c --- /dev/null +++ b/server/src/lib/active-run-slots.ts @@ -0,0 +1,20 @@ +export interface ActiveRunSlotBounds { + runsPerSlot: number; + floor: number; + ceiling: number; +} + +export function assertActiveRunSlotBounds(bounds: ActiveRunSlotBounds): ActiveRunSlotBounds { + const { runsPerSlot, floor, ceiling } = bounds; + if (!Number.isSafeInteger(runsPerSlot) || runsPerSlot < 1 + || !Number.isSafeInteger(floor) || floor < 1 + || !Number.isSafeInteger(ceiling) || ceiling < floor) { + throw new Error("invalid_active_run_slot_bounds"); + } + return bounds; +} + +export function slotsForActiveRuns(activeRuns: number, bounds: ActiveRunSlotBounds): number { + const runs = Number.isFinite(activeRuns) && activeRuns > 0 ? Math.floor(activeRuns) : 0; + return Math.min(bounds.ceiling, Math.max(bounds.floor, Math.ceil(runs / bounds.runsPerSlot))); +} diff --git a/server/src/services/object-byte-journal.ts b/server/src/services/object-byte-journal.ts index f9a5b3003ae..88d12fb1067 100644 --- a/server/src/services/object-byte-journal.ts +++ b/server/src/services/object-byte-journal.ts @@ -3,6 +3,8 @@ import { runSharedStorageWork, sendThroughStorageOutage } from "../storage/stora import { z } from "zod"; import { setTimeout as delay } from "node:timers/promises"; import type { StorageProvider } from "../storage/types.js"; +import { S3_CLIENT_MAX_SOCKETS } from "../storage/s3-client-limits.js"; +import { assertActiveRunSlotBounds, slotsForActiveRuns } from "../lib/active-run-slots.js"; const MAX_INDEX_LEVEL = 52; const MAX_OBJECT_BYTES = 4 * 1024 * 1024; @@ -217,20 +219,90 @@ export function isObjectJournalBusyError(error: unknown): error is ObjectJournal } export type ObjectJournalIOPriority = "background" | "interactive"; -const MAX_ACTIVE_IO = 8; +export const OBJECT_JOURNAL_READ_IO_RESERVE = 4; +export const OBJECT_JOURNAL_IO_CEILING = Math.floor(S3_CLIENT_MAX_SOCKETS / 2); +export const OBJECT_JOURNAL_UPLOAD_SLOTS = assertActiveRunSlotBounds({ + runsPerSlot: 10, + floor: 4, + ceiling: OBJECT_JOURNAL_IO_CEILING - OBJECT_JOURNAL_READ_IO_RESERVE, +}); const MAX_WAITING_INTERACTIVE_IO = 32; +let openWriters = 0; let activeIO = 0; const waitingIO: Record void>> = { background: [], interactive: [], }; +export function objectJournalUploadSlots() { + return slotsForActiveRuns(openWriters, OBJECT_JOURNAL_UPLOAD_SLOTS); +} +export function objectJournalIOSlots() { + return objectJournalUploadSlots() + OBJECT_JOURNAL_READ_IO_RESERVE; +} +export function trackObjectJournalWriter(): () => void { + openWriters++; + grantWaitingUploads(); + grantWaitingIO(); + let released = false; + return () => { + if (released) return; + released = true; + openWriters--; + }; +} +let activeUploads = 0; +const uploadWaiters: Array<() => void> = []; +export function objectJournalUploadActivity() { + return { active: activeUploads, waiting: uploadWaiters.length }; +} +function grantWaitingUploads() { + while (activeUploads < objectJournalUploadSlots()) { + const next = uploadWaiters.shift(); + if (!next) return; + activeUploads++; + next(); + } +} +export async function acquireObjectJournalUpload(signal?: AbortSignal): Promise<() => void> { + signal?.throwIfAborted(); + if (activeUploads >= objectJournalUploadSlots()) + await new Promise((resolve, reject) => { + const enter = () => { + signal?.removeEventListener("abort", abort); + resolve(); + }; + const abort = () => { + const index = uploadWaiters.indexOf(enter); + if (index !== -1) uploadWaiters.splice(index, 1); + reject(signal!.reason); + }; + signal?.addEventListener("abort", abort, { once: true }); + uploadWaiters.push(enter); + }); + else activeUploads++; + let released = false; + return () => { + if (released) return; + released = true; + activeUploads--; + grantWaitingUploads(); + }; +} +function grantWaitingIO() { + while (activeIO < objectJournalIOSlots()) { + const next = waitingIO.background.shift() ?? waitingIO.interactive.shift(); + if (!next) return; + activeIO++; + next(); + } +} async function objectIO( operation: () => Promise, priority: ObjectJournalIOPriority = "background", signal?: AbortSignal, ): Promise { signal?.throwIfAborted(); - if (activeIO >= MAX_ACTIVE_IO) { + if (activeIO >= objectJournalIOSlots()) { const queue = waitingIO[priority]; if (priority === "interactive" && queue.length >= MAX_WAITING_INTERACTIVE_IO) throw new ObjectJournalBusyError(); @@ -251,9 +323,8 @@ async function objectIO( try { return await operation(); } finally { - const next = waitingIO.background.shift() ?? waitingIO.interactive.shift(); - if (next) next(); - else activeIO--; + activeIO--; + grantWaitingIO(); } } diff --git a/server/src/services/object-run-log-store.test.ts b/server/src/services/object-run-log-store.test.ts index 04ccc8b2f50..cc817d72f02 100644 --- a/server/src/services/object-run-log-store.test.ts +++ b/server/src/services/object-run-log-store.test.ts @@ -2,7 +2,9 @@ import { createHash } from "node:crypto"; import { Readable } from "node:stream"; import { describe, expect, it, vi } from "vitest"; import type { StorageProvider } from "../storage/types.js"; -import { ObjectJournalBusyError } from "./object-byte-journal.js"; +import { S3_CLIENT_MAX_SOCKETS } from "../storage/s3-client-limits.js"; +import { OBJECT_JOURNAL_IO_CEILING, OBJECT_JOURNAL_READ_IO_RESERVE, OBJECT_JOURNAL_UPLOAD_SLOTS, ObjectJournalBusyError, + objectJournalIOSlots, objectJournalUploadActivity, objectJournalUploadSlots } from "./object-byte-journal.js"; import { createObjectRunLogStore } from "./object-run-log-store.js"; async function readBody(body: AsyncIterable) { @@ -569,4 +571,103 @@ describe("S3 run log store", () => { await Promise.allSettled([...polls, excess, ...(internal ? [internal] : [])]); } }); + + it("keeps the object storage sockets the journal may use within the storage client's budget", () => { + expect(OBJECT_JOURNAL_UPLOAD_SLOTS.ceiling + OBJECT_JOURNAL_READ_IO_RESERVE).toBeLessThanOrEqual(OBJECT_JOURNAL_IO_CEILING); + expect(OBJECT_JOURNAL_IO_CEILING).toBeLessThanOrEqual(S3_CLIENT_MAX_SOCKETS / 2); + }); + + async function uploadsInFlightWithOpenRuns(runs: number) { + const m = memory(), + store = createObjectRunLogStore({ provider: m.provider, keyPrefix: "logs", flushMs: 30_000, + assertWriterAuthority: async () => undefined }); + const slotsBefore = objectJournalUploadSlots(); + const handles = []; + for (let run = 0; run < runs; run++) handles.push(await store.begin({ ...input, runId: `run-${run}` })); + const put = m.provider.putObjectConditional!.bind(m.provider); + let open!: () => void; + const gate = new Promise(resolve => { open = resolve; }); + let inFlight = 0, peak = 0; + m.provider.putObjectConditional = async value => { + inFlight++; + peak = Math.max(peak, inFlight); + try { + await gate; + return await put(value); + } finally { + inFlight--; + } + }; + const appended = await Promise.all(handles.map(handle => store.enqueue!(handle, event(1)))); + const flushes = handles.map(handle => store.flush!(handle)); + await vi.waitFor(() => expect(inFlight).toBe(Math.min(runs, objectJournalUploadSlots()))); + await new Promise(resolve => setImmediate(resolve)); + const waiting = { slotsBefore, slots: objectJournalUploadSlots(), ioSlots: objectJournalIOSlots() }; + open(); + await Promise.all(flushes); + await Promise.all(appended.map(entry => entry.committed)); + await Promise.all(handles.map(handle => store.finalize(handle, { computeDigest: false }))); + return { ...waiting, peak, slotsAfterFinalize: objectJournalUploadSlots() }; + } + + it("gives a log upload slot to each ten open runs, so 200 runs upload at least twenty logs at once", async () => { + const result = await uploadsInFlightWithOpenRuns(200); + expect(result.slotsBefore).toBe(OBJECT_JOURNAL_UPLOAD_SLOTS.floor); + expect(result.peak).toBeGreaterThanOrEqual(20); + expect(result.peak).toBeLessThanOrEqual(result.ioSlots); + }); + + it("caps log uploads at the storage budget with 1,000 open runs and finishes every run's upload", async () => { + const result = await uploadsInFlightWithOpenRuns(1_000); + expect(result.slots).toBe(OBJECT_JOURNAL_UPLOAD_SLOTS.ceiling); + expect(result.peak).toBeLessThanOrEqual(result.ioSlots); + expect(result.ioSlots).toBeLessThanOrEqual(S3_CLIENT_MAX_SOCKETS / 2); + expect(result.slotsAfterFinalize).toBeLessThan(result.slots); + }); + + it("starts waiting uploads when runs open and starts no new upload after runs close until the count is under the limit", async () => { + const options = { keyPrefix: "logs", flushMs: 30_000, assertWriterAuthority: async () => undefined }; + const busy = memory(), idle = memory(); + const busyStore = createObjectRunLogStore({ ...options, provider: busy.provider }); + const idleStore = createObjectRunLogStore({ ...options, provider: idle.provider }); + const busyHandles = []; + for (let run = 0; run < 40; run++) busyHandles.push(await busyStore.begin({ ...input, runId: `busy-${run}` })); + const slotsBefore = objectJournalUploadSlots(); + const put = busy.provider.putObjectConditional!.bind(busy.provider); + const held: Array<() => void> = []; + busy.provider.putObjectConditional = async value => { + await new Promise(resolve => held.push(resolve)); + return put(value); + }; + const appended = await Promise.all(busyHandles.map(handle => busyStore.enqueue!(handle, event(1)))); + const flushes = busyHandles.map(handle => busyStore.flush!(handle)); + await vi.waitFor(() => expect(objectJournalUploadActivity().active).toBe(slotsBefore)); + expect(objectJournalUploadActivity().waiting).toBe(40 - slotsBefore); + + const idleHandles = []; + for (let run = 0; run < 160; run++) idleHandles.push(await idleStore.begin({ ...input, runId: `idle-${run}` })); + const grown = objectJournalUploadSlots(); + expect(grown).toBeGreaterThanOrEqual(slotsBefore + 16); + expect(objectJournalUploadActivity().active).toBe(grown); + await vi.waitFor(() => expect(held).toHaveLength(grown)); + + const closing = Promise.all(idleHandles.map(handle => idleStore.finalize(handle, { computeDigest: false }))); + let previous = grown, shrankWhileBusy = false; + while (held.length || objectJournalUploadActivity().waiting || objectJournalUploadSlots() > slotsBefore) { + held.shift()?.(); + await new Promise(resolve => setImmediate(resolve)); + await new Promise(resolve => setImmediate(resolve)); + const { active } = objectJournalUploadActivity(), limit = objectJournalUploadSlots(); + expect(active).toBeLessThanOrEqual(Math.max(previous, limit)); + if (active > limit && limit === slotsBefore) shrankWhileBusy = true; + previous = active; + } + await closing; + busy.provider.putObjectConditional = put; + await Promise.all(flushes); + await Promise.all(appended.map(entry => entry.committed)); + await Promise.all(busyHandles.map(handle => busyStore.finalize(handle, { computeDigest: false }))); + expect(shrankWhileBusy).toBe(true); + expect(objectJournalUploadActivity()).toEqual({ active: 0, waiting: 0 }); + }); }); diff --git a/server/src/services/object-run-log-store.ts b/server/src/services/object-run-log-store.ts index 154c996d320..8ffee47f9ad 100644 --- a/server/src/services/object-run-log-store.ts +++ b/server/src/services/object-run-log-store.ts @@ -2,7 +2,8 @@ import { beginIdleTrackedWork } from "./task-admission.js"; import { createHash, randomUUID } from "node:crypto"; import type { StorageProvider } from "../storage/types.js"; import type { RunLogStore, RunLogHandle, RunLogBeginInput } from "./run-log-store.js"; -import { createObjectByteJournal, ObjectJournalSourcePrefixProofUnavailableError, type ObjectJournalIOPriority, type ObjectJournalWriter } from "./object-byte-journal.js"; +import { createObjectByteJournal, ObjectJournalSourcePrefixProofUnavailableError, acquireObjectJournalUpload, trackObjectJournalWriter, + type ObjectJournalIOPriority, type ObjectJournalWriter } from "./object-byte-journal.js"; import { MAX_RUN_LOG_RECORD_BYTES } from "./run-log-record-limits.js"; import { notFound } from "../errors.js"; @@ -39,6 +40,7 @@ interface WriterState { admitting: Set>; failure?: unknown; signal?: AbortSignal; + releaseUploadDemand: () => void; sourcePrefixVerified: boolean; coverageEligible: boolean; admittedSourceCursor: number; @@ -65,31 +67,6 @@ export function createObjectRunLogStore(options: ObjectRunLogStoreOptions): RunL for (const wake of capacityWaiters) wake(); capacityWaiters.clear(); }; - let activeUploads = 0; - const uploadWaiters: Array<() => void> = []; - async function acquireUpload(signal?: AbortSignal) { - signal?.throwIfAborted(); - if (activeUploads >= 4) - await new Promise((resolve, reject) => { - const enter = () => { - signal?.removeEventListener("abort", abort); - resolve(); - }; - const abort = () => { - const index = uploadWaiters.indexOf(enter); - if (index !== -1) uploadWaiters.splice(index, 1); - reject(signal!.reason); - }; - signal?.addEventListener("abort", abort, { once: true }); - uploadWaiters.push(enter); - }); - else activeUploads++; - return () => { - const next = uploadWaiters.shift(); - if (next) next(); - else activeUploads--; - }; - } function journal(logRef: string, readPriority?: ObjectJournalIOPriority) { if (!/^[a-zA-Z0-9._-]+\/[a-zA-Z0-9._-]+\/[a-zA-Z0-9._-]+$/.test(logRef)) throw new Error("Invalid object log reference"); @@ -123,7 +100,7 @@ export function createObjectRunLogStore(options: ObjectRunLogStoreOptions): RunL state.flushing = (async () => { let release: (() => void) | undefined; try { - release = await acquireUpload(state.signal); + release = await acquireObjectJournalUpload(state.signal); state.writer = await state.journal.append( state.writer, bytes, @@ -278,6 +255,7 @@ export function createObjectRunLogStore(options: ObjectRunLogStoreOptions): RunL closed: false, admitting: new Set(), signal: input.signal, + releaseUploadDemand: trackObjectJournalWriter(), sourcePrefixVerified, coverageEligible: true, admittedSourceCursor: writer.head.sourceCursor, @@ -352,6 +330,7 @@ export function createObjectRunLogStore(options: ObjectRunLogStoreOptions): RunL return { bytes: null, sha256: null, compressed: false }; } finally { writers.delete(handle); + state.releaseUploadDemand(); } }, async read(handle, opts = {}) { diff --git a/server/src/storage/s3-client-limits.ts b/server/src/storage/s3-client-limits.ts new file mode 100644 index 00000000000..35bad7c0a3c --- /dev/null +++ b/server/src/storage/s3-client-limits.ts @@ -0,0 +1 @@ +export const S3_CLIENT_MAX_SOCKETS = 50; diff --git a/server/src/storage/s3-provider.ts b/server/src/storage/s3-provider.ts index 3d0cdd8e531..e2c3bab8559 100644 --- a/server/src/storage/s3-provider.ts +++ b/server/src/storage/s3-provider.ts @@ -17,6 +17,7 @@ import { sendThroughStorageOutage } from "./storage-unavailable.js"; import { addAbortSignal, Readable } from "node:stream"; import { retryS3RequestTimeouts } from "./s3-request-timeout-retry.js"; import { refreshS3CredentialsOnRejection } from "./s3-credential-refresh.js"; +import { S3_CLIENT_MAX_SOCKETS } from "./s3-client-limits.js"; import type { StorageProvider, GetObjectResult, HeadObjectResult } from "./types.js"; import { notFound, unprocessable } from "../errors.js"; @@ -138,6 +139,10 @@ export function createS3StorageProvider(config: S3ProviderConfig): StorageProvid region, endpoint: config.endpoint, forcePathStyle: Boolean(config.forcePathStyle), + requestHandler: { + httpAgent: { maxSockets: S3_CLIENT_MAX_SOCKETS }, + httpsAgent: { maxSockets: S3_CLIENT_MAX_SOCKETS }, + }, }); retryS3RequestTimeouts(client); refreshS3CredentialsOnRejection(client);