diff --git a/doc/plans/2026-11-13-thousand-concurrent-agents.md b/doc/plans/2026-11-13-thousand-concurrent-agents.md index 825f67dcdfa..24e850f2a4b 100644 --- a/doc/plans/2026-11-13-thousand-concurrent-agents.md +++ b/doc/plans/2026-11-13-thousand-concurrent-agents.md @@ -362,9 +362,12 @@ of other servers keep their records. `scripts/scale-agents-ramp.ts` runs real agents on a hosted instance through the product API only. It ramps concurrency in steps, holds each step, stops at -the first failed criterion, and always cleans up. It has run only against the -in-process fake API of its tests (`scripts/scale-ramp/scale-ramp.test.ts`). -A run on a real instance needs the owner's go-ahead. +the first failed criterion, and always cleans up. A run on a real instance +needs the owner's go-ahead. Its first real run, on 2026-11-15, held a +20-minute step with nothing running: it bound the agents to a plugin +Runner-task environment (Paperclip Sandbox (Daytona)), which needs a managed +AI connection, so every agent stayed in setup with that reason and every run +stayed queued. The harness now checks this before it creates the rest. **What one run does.** It creates a company named `Scale ramp `, or uses `--company ` (a Cloud-managed instance refuses company creation, so @@ -377,6 +380,22 @@ concurrency. Each task is the same small job: stream a sentence, run `pwd` and `ls`, write `scale-.txt` in the workspace, read it back, post a `CHECKPOINT` comment, post a `DONE` comment and mark the task done. +**Start checks.** A new agent cannot run until it finishes setup +(`lifecycleState` `ready`). Before each step the harness waits for its agents +to finish setup, at most `--agent-ready-minutes` (default 10). It stops when +an agent reports a setup error for 60 s, and prints the agent's state and +Paperclip's reason. Before the first step it creates one agent and one task, +and creates the rest only after that task's run starts. If no run starts +within `--start-timeout-minutes` (default 5) of the first task, it stops and +prints the task status, the run status and error, and the agent's state. +Every stop cleans up. + +**Environment.** Use a `sandbox`-driver environment (Paperclip Sandbox), +which takes the provider key the harness stores. A plugin environment that +runs Runner tasks (Paperclip Sandbox (Daytona), Paperclip Computer, +Paperclip Sandbox (Unikraft)) runs only agents with a managed AI connection. +The harness does not set one up, so its agents never finish setup there. + **Run it.** ```sh diff --git a/scripts/scale-ramp/config.ts b/scripts/scale-ramp/config.ts index 2165cbbe75d..65db8760d38 100644 --- a/scripts/scale-ramp/config.ts +++ b/scripts/scale-ramp/config.ts @@ -49,6 +49,8 @@ export interface Options { pollMs: number; boardProbeMs: number; taskMinutes: number; + startTimeoutMs: number; + agentReadyTimeoutMs: number; maxTasks: number | null; companyId: string | null; environmentId: string | null; @@ -141,6 +143,8 @@ export function parseOptions(argv: readonly string[], env: NodeJS.ProcessEnv, no "poll-seconds": { type: "string" }, "board-probe-seconds": { type: "string" }, "task-minutes": { type: "string" }, + "start-timeout-minutes": { type: "string" }, + "agent-ready-minutes": { type: "string" }, "max-tasks": { type: "string" }, company: { type: "string" }, environment: { type: "string" }, @@ -180,6 +184,8 @@ export function parseOptions(argv: readonly string[], env: NodeJS.ProcessEnv, no pollMs: number(values["poll-seconds"], "--poll-seconds", 15, 1, 300) * 1_000, boardProbeMs: number(values["board-probe-seconds"], "--board-probe-seconds", 10, 1, 300) * 1_000, taskMinutes: number(values["task-minutes"], "--task-minutes", 3, 0.5, 120), + startTimeoutMs: number(values["start-timeout-minutes"], "--start-timeout-minutes", 5, 1, 60) * 60_000, + agentReadyTimeoutMs: number(values["agent-ready-minutes"], "--agent-ready-minutes", 10, 1, 60) * 60_000, maxTasks: values["max-tasks"] === undefined ? null : number(values["max-tasks"], "--max-tasks", 0, 1, 1_000_000), companyId: values.company ?? env.PAPERCLIP_SCALE_COMPANY_ID ?? null, environmentId: values.environment ?? env.PAPERCLIP_SCALE_ENVIRONMENT_ID ?? null, diff --git a/scripts/scale-ramp/fake-paperclip.ts b/scripts/scale-ramp/fake-paperclip.ts index efd17e9d02a..0dcddcac131 100644 --- a/scripts/scale-ramp/fake-paperclip.ts +++ b/scripts/scale-ramp/fake-paperclip.ts @@ -19,6 +19,8 @@ export interface FakeConfig { token: string; origin: string; cloudManaged: boolean; + agentReadyDelayMs: number | ((index: number) => number); + agentSetupError: string | null; startDelayMs: number; runDurationMs: number; capacity: number; @@ -63,6 +65,7 @@ export class FakePaperclip { readonly memberCompanyIds: string[] = []; readonly environmentId = "00000000-0000-4000-8000-0000000000e1"; private seq = 0; + private agentsCreated = 0; private counts = new Map(); readonly clock: VirtualClock; @@ -73,6 +76,8 @@ export class FakePaperclip { token: "board-token-value", origin: "https://scale1.test.paperclip.app", cloudManaged: false, + agentReadyDelayMs: 3_000, + agentSetupError: null, startDelayMs: 2_000, runDurationMs: 60_000, capacity: Number.POSITIVE_INFINITY, @@ -115,8 +120,24 @@ export class FakePaperclip { return [...this.runs.values()].filter(run => ["queued", "running"].includes(run.status)); } + private invokable(agentId: string): boolean { + const agent = this.agents.get(agentId); + return Boolean(agent && agent.lifecycleState === "ready" && agent.status === "idle"); + } + advance(): void { const now = this.clock.now(); + for (const agent of this.agents.values()) { + const delay = typeof this.config.agentReadyDelayMs === "function" ? this.config.agentReadyDelayMs(agent.index) : this.config.agentReadyDelayMs; + if (!["preparing", "verifying"].includes(agent.lifecycleState) || now < agent.createdAt + delay) continue; + if (this.config.agentSetupError) { + agent.lifecycleState = "verifying"; + agent.lifecycleError = this.config.agentSetupError; + } else { + agent.lifecycleState = "ready"; + agent.status = "idle"; + } + } for (const run of [...this.runs.values()].sort((a, b) => a.createdAt - b.createdAt)) { if (run.status === "running" && run.startedAt !== null && now >= run.startedAt + this.config.runDurationMs) { run.finishedAt = run.startedAt + this.config.runDurationMs; @@ -131,7 +152,7 @@ export class FakePaperclip { } } for (const run of [...this.runs.values()].sort((a, b) => a.createdAt - b.createdAt)) { - if (run.status !== "queued" || now < run.createdAt + this.config.startDelayMs) continue; + if (run.status !== "queued" || now < run.createdAt + this.config.startDelayMs || !this.invokable(run.agentId)) continue; if (this.running() >= this.config.capacity) break; run.status = "running"; run.startedAt = Math.max(run.createdAt + this.config.startDelayMs, now - 1_000); @@ -145,6 +166,7 @@ export class FakePaperclip { return { id: run.id, agentId: run.agentId, issueId: run.issueId, status: run.status, createdAt: iso(run.createdAt), startedAt: iso(run.startedAt), finishedAt: iso(run.finishedAt), costUsd: run.status === "succeeded" || run.status === "failed" ? this.config.costPerRunUsd : null, + error: null, errorCode: null, }; } @@ -208,7 +230,8 @@ export class FakePaperclip { if (c === "agents") { if (method === "GET") return json(200, [...this.agents.values()].filter(agent => agent.companyId === companyId)); const id = this.id(); - this.agents.set(id, { id, companyId, name: body.name, status: "idle", lifecycleState: null, payload: body }); + this.agents.set(id, { id, companyId, name: body.name, status: "paused", lifecycleState: "preparing", lifecycleHolds: [], lifecycleError: null, + pauseReason: null, createdAt: now, index: this.agentsCreated++, payload: body }); return json(201, this.agents.get(id)); } if (c === "issues") { @@ -217,7 +240,7 @@ export class FakePaperclip { this.issues.set(id, { id, companyId, title: body.title, description: body.description, status: "todo", assigneeAgentId: body.assigneeAgentId, projectId: body.projectId ?? null, hiddenAt: null, interruption: null }); const agent = this.agents.get(body.assigneeAgentId); - if (agent && agent.status !== "paused" && agent.status !== "terminated") { + if (agent && agent.status !== "terminated") { const runId = this.id(); this.runs.set(runId, { id: runId, agentId: body.assigneeAgentId, issueId: id, status: "queued", createdAt: now, startedAt: null, finishedAt: null, willFail: false }); } diff --git a/scripts/scale-ramp/ramp.ts b/scripts/scale-ramp/ramp.ts index 9dc68a00faa..848ac9c267c 100644 --- a/scripts/scale-ramp/ramp.ts +++ b/scripts/scale-ramp/ramp.ts @@ -2,7 +2,7 @@ import { readMetrics, type Clock, type Fetcher, type PaperclipClient } from "./c import type { Options, Step } from "./config.ts"; import { evaluate, firstFailure, type CriterionResult, type ServerMetrics, type StepMetrics } from "./criteria.ts"; import { LIVE_RUN_STATUSES } from "./cleanup.ts"; -import { ensureAgents } from "./setup.ts"; +import { AgentsNotReady, describeAgent, ensureAgents, waitForAgentsReady } from "./setup.ts"; import { distribution } from "./stats.ts"; import { mapLimit, taskTag, type RunState, type StateStore } from "./state.ts"; import { runTiming, taskBrief, taskNonce, taskTitle, type RunEvent } from "./workload.ts"; @@ -19,6 +19,7 @@ export const BOARD_MIX = [ const FAILED_RUN = new Set(["failed", "timed_out"]); const RESTING_NOT_DONE = new Set(["cancelled", "blocked", "in_review"]); +const ENDED_RUN = new Set(["failed", "timed_out", "cancelled", "succeeded", "interrupted"]); const RUN_PAGE = 1_000; const IDS_PER_READ = 100; const RETRY_CHECK_AFTER_MS = 60_000; @@ -51,6 +52,9 @@ interface RunRecord { finishedAt: number | null; costUsd: number; sampled: boolean; + error: string | null; + errorCode: string | null; + liveness: string | null; } interface StepAccumulator { @@ -216,14 +220,16 @@ export class Ramp { break; } log(`Step ${step.index + 1}/${options.steps.length}: ${step.concurrency} agents for ${Math.round(step.holdMs / 60_000)} min`); - await ensureAgents({ api: this.deps.api, options, state: this.deps.state, store: this.deps.store, count: step.concurrency }); const accumulator: StepAccumulator = { step, startedAt: clock.now(), endedAt: null, runningSamples: [], boardMs: [], boardRequests: 0, boardFailures: 0, serverSamples: [], ttftMs: [], firstActivityMs: [], sampled: 0 }; this.accumulators.push(accumulator); - this.nextPollAt = 0; - this.nextBoardAt = 0; - const until = accumulator.startedAt + step.holdMs; - stop = await this.loop(until, step.concurrency); + stop = await this.prepareStep(step); + if (!stop) { + this.nextPollAt = 0; + this.nextBoardAt = 0; + accumulator.startedAt = clock.now(); + stop = await this.loop(accumulator.startedAt + step.holdMs, step.concurrency); + } accumulator.endedAt = clock.now(); const atEnd = evaluate(this.metrics(step.index), options.thresholds); results.push({ index: step.index, concurrency: step.concurrency, holdMs: step.holdMs, metrics: this.metrics(step.index), criteriaAtEnd: atEnd, criteriaFinal: atEnd }); @@ -253,6 +259,97 @@ export class Ramp { }; } + private async prepareStep(step: Step): Promise { + try { + if (step.index === 0) { + const stop = await this.preflight(); + if (stop) return stop; + } + if (this.deps.shouldStop()) return { stepIndex: step.index, reason: "interrupted before the step", criterion: null }; + await ensureAgents({ api: this.deps.api, options: this.deps.options, state: this.deps.state, store: this.deps.store, count: step.concurrency }); + return await this.awaitAgents(step.concurrency, step.index); + } catch (error) { + if (!(error instanceof AgentsNotReady)) throw error; + this.deps.log(`Stopping: ${error.message}`); + return { stepIndex: step.index, reason: error.message, criterion: "agents_ready" }; + } + } + + private async awaitAgents(count: number, stepIndex: number): Promise { + const { api, clock, options, state, log, shouldStop } = this.deps; + let stop: StopReason | null = null; + await waitForAgentsReady({ api, clock, companyId: this.companyId, agentIds: state.agentIds.slice(0, count), timeoutMs: options.agentReadyTimeoutMs, + pollMs: Math.min(options.pollMs, 5_000), shouldStop, log, + tick: async () => { + if (!this.tasks.size) return false; + await this.poll(false); + stop = this.immediateStop(stepIndex); + return stop !== null; + } }); + return stop; + } + + private async preflight(): Promise { + const { api, clock, options, state, store, log } = this.deps; + await ensureAgents({ api, options, state, store, count: 1 }); + const notReady = await this.awaitAgents(1, 0); + if (notReady) return notReady; + if (this.deps.shouldStop()) return { stepIndex: 0, reason: "interrupted before the step", criterion: null }; + log(`Preflight: one task on one agent must start within ${Math.round(options.startTimeoutMs / 60_000)} min before the rest are created`); + await this.topUp(1, 0, Number.POSITIVE_INFINITY); + const task = [...this.tasks.values()][0]; + if (!task) return { stepIndex: 0, reason: "the preflight task was not created", criterion: "run_started" }; + for (;;) { + if (this.deps.shouldStop()) return { stepIndex: 0, reason: "interrupted", criterion: null }; + await this.poll(); + const immediate = this.immediateStop(0); + if (immediate) return immediate; + if (this.anyStarted()) { + log(`Preflight passed: the first task started ${Math.round((clock.now() - task.createdAt) / 1_000)} s after it was created`); + this.current().runningSamples.length = 0; + return null; + } + const ended = this.runsOf(task.id).find(run => run.finishedAt !== null || ENDED_RUN.has(run.status)); + if (task.outcome !== "open" || ended) { + return this.stall(0, task, `the preflight task ended before any run started`); + } + const stop = await this.noStartStop(0); + if (stop) return stop; + await clock.sleep(options.pollMs); + } + } + + private anyStarted(): boolean { + for (const run of this.runs.values()) if (run.startedAt !== null || run.status === "running") return true; + return false; + } + + private async noStartStop(stepIndex: number): Promise { + const { clock, options } = this.deps; + if (!this.tasks.size || this.anyStarted()) return null; + const first = [...this.tasks.values()].sort((a, b) => a.createdAt - b.createdAt)[0]!; + if (clock.now() - first.createdAt < options.startTimeoutMs) return null; + return this.stall(stepIndex, first, `no run started within ${Math.round(options.startTimeoutMs / 60_000)} min of the first task`); + } + + private async stall(stepIndex: number, task: TaskRecord, headline: string): Promise { + const { api, log } = this.deps; + const [view, agents] = await Promise.all([ + api.get(`/api/issues/${task.id}`).catch(() => null), + api.get(`/api/companies/${this.companyId}/agents`).catch(() => [] as Json[]), + ]); + const run = this.runsOf(task.id).at(-1); + const agent = agents.find(candidate => String(candidate.id) === task.agentId); + const runLine = run + ? `its run ${run.id} is ${run.status}${run.errorCode ? ` (${run.errorCode})` : ""}${run.error ? `: ${run.error.slice(0, 300)}` : ""}${run.liveness ? `; liveness: ${run.liveness}` : ""}` + : "Paperclip lists no run for it"; + const blocker = view?.executionBlocker ? `; execution blocker: ${JSON.stringify(view.executionBlocker).slice(0, 300)}` : ""; + const reason = `${headline}. Task ${task.nonce} is ${view?.status ?? task.status}` + + `${blocker}; ${runLine}; ${agent ? describeAgent(agent) : `agent ${task.agentId} is not listed`}`; + log(`Stopping: ${reason}`); + return { stepIndex, reason, criterion: "run_started" }; + } + private async loop(until: number, concurrency: number, draining = false): Promise { const { clock, options } = this.deps; const stepIndex = this.current().step.index; @@ -265,7 +362,7 @@ export class Ramp { if (clock.now() >= this.nextPollAt) { this.nextPollAt = clock.now() + options.pollMs; await this.poll(); - const stop = this.immediateStop(stepIndex); + const stop = this.immediateStop(stepIndex) ?? await this.noStartStop(stepIndex); if (stop) return stop; if (!draining) await this.topUp(concurrency, stepIndex, until); else if (this.openTasks().length === 0) return null; @@ -337,13 +434,14 @@ export class Ramp { } } - private async poll(): Promise { + private async poll(recordSample = true): Promise { await this.readRuns(); await this.readTasks(); await this.checkRetry(); await this.sampleEvents(); let running = 0; for (const run of this.runs.values()) if (run.status === "running") running += 1; + if (!recordSample) return; this.current().runningSamples.push(running); if (this.deps.options.metricsUrl) { const sample = await readMetrics(this.deps.fetcher, this.deps.options.metricsUrl, this.deps.metricsToken, this.deps.clock); @@ -362,6 +460,9 @@ export class Ramp { this.runs.set(id, { id, taskId, agentId: String(row.agentId), status: String(row.status), createdAt: ms(row.createdAt), startedAt: ms(row.startedAt), finishedAt: ms(row.finishedAt), costUsd: num(row.costUsd) ?? previous?.costUsd ?? 0, sampled: previous?.sampled ?? false, + error: typeof row.error === "string" && row.error ? row.error : null, + errorCode: typeof row.errorCode === "string" && row.errorCode ? row.errorCode : null, + liveness: typeof row.livenessReason === "string" && row.livenessReason ? row.livenessReason : null, }); const own = this.runsByTask.get(taskId) ?? new Set(); own.add(id); diff --git a/scripts/scale-ramp/scale-ramp.test.ts b/scripts/scale-ramp/scale-ramp.test.ts index 6a34655e617..3eab4e9ac87 100644 --- a/scripts/scale-ramp/scale-ramp.test.ts +++ b/scripts/scale-ramp/scale-ramp.test.ts @@ -349,6 +349,176 @@ describe("ramp against the fake API", () => { assert.equal(result.cleanup!.activeSandboxLeases, 0); }); + it("stops within minutes with Paperclip's reason when the agents never finish setup", async () => { + const out = await tempDir(); + const reason = "Execution configuration check failed. This agent's environment requires a managed AI connection and an isolated Runner home. " + + "Check both in Configuration, then retry. Automatic verification is paused. Update Configuration or choose Retry now."; + const h = harness({ agentSetupError: reason }); + const started = h.clock.now(); + const code = await main(baseArgs(out, ["--steps", "10,50", "--hold-minutes", "20", "--confirm-spend", "100"]), h.deps); + const result = await readResult(out); + assert.equal(code, EXIT.fail); + assert.equal(result.stop!.criterion, "agents_ready"); + assert.ok(result.stop!.reason.includes(reason), result.stop!.reason); + assert.match(result.stop!.reason, /setup state verifying/); + assert.ok(h.clock.now() - started < 5 * 60_000, "the step is not held while nothing can run"); + const agents = h.fake.requests.filter(request => request.method === "POST" && new URL(request.url).pathname.endsWith("/agents")); + assert.equal(agents.length, 1, "only the preflight agent is created"); + assert.equal(h.fake.issues.size, 0); + assertCleanedUp(h.fake); + }); + + it("stops at the preflight with the task, run and agent state when the first task never starts", async () => { + const out = await tempDir(); + const h = harness({ capacity: 0 }); + const started = h.clock.now(); + const code = await main(baseArgs(out, ["--steps", "10,50", "--hold-minutes", "20", "--confirm-spend", "100", "--start-timeout-minutes", "3"]), h.deps); + const result = await readResult(out); + assert.equal(code, EXIT.fail); + assert.equal(result.stop!.criterion, "run_started"); + assert.match(result.stop!.reason, /no run started within 3 min/); + assert.match(result.stop!.reason, /is todo/); + assert.match(result.stop!.reason, /is queued/); + assert.match(result.stop!.reason, /setup state ready/); + assert.ok(h.clock.now() - started < 5 * 60_000); + assert.equal(h.fake.issues.size, 1, "only the preflight task is created"); + assert.equal(result.cleanup!.runsCancelled, 1); + assertCleanedUp(h.fake); + }); + + it("creates the rest of the step only after the preflight task starts", async () => { + const out = await tempDir(); + const h = harness(); + await main(baseArgs(out, ["--steps", "3"]), h.deps); + const posts = h.fake.requests.map(request => request.method === "POST" ? new URL(request.url).pathname : ""); + const firstTask = posts.findIndex(route => route.endsWith("/issues")); + const secondAgent = posts.findIndex((route, index) => route.endsWith("/agents") && posts.slice(0, index).some(earlier => earlier.endsWith("/agents"))); + assert.ok(firstTask >= 0 && secondAgent > firstTask, "the second agent is created after the first task"); + const result = await readResult(out); + assert.equal(result.steps[0]!.metrics.peakRunning, 3); + }); + + const preflightCases: Array<{ name: string; mutate: (issue: Record, run: { status: string; finishedAt: number | null }, now: number) => void; + criterion: string; reason: RegExp }> = [ + { name: "cancelled", criterion: "run_started", reason: /ended before any run started.*is cancelled.*is cancelled/, + mutate: (issue, run, now) => { issue.status = "cancelled"; run.status = "cancelled"; run.finishedAt = now; } }, + { name: "blocked", criterion: "run_started", reason: /ended before any run started.*is blocked.*is queued/, + mutate: issue => { issue.status = "blocked"; } }, + { name: "failed before it started", criterion: "failed_runs_zero", reason: /failed_runs_zero failed/, + mutate: (_issue, run, now) => { run.status = "failed"; run.finishedAt = now; } }, + ]; + for (const preflight of preflightCases) { + it(`stops at once and creates nothing more when the preflight task is ${preflight.name}`, async () => { + const out = await tempDir(); + const h = harness({ capacity: 0 }); + const original = h.deps.fetcher; + h.deps.fetcher = async (url, init) => { + const response = await original(url, init); + if (init.method === "POST" && new URL(url).pathname.endsWith("/issues")) { + const issue = h.fake.issues.get((await response.clone().json() as { id: string }).id)!; + const run = [...h.fake.runs.values()].find(candidate => candidate.issueId === issue.id)!; + preflight.mutate(issue, run, h.clock.now()); + } + return response; + }; + const started = h.clock.now(); + const code = await main(baseArgs(out, ["--steps", "3", "--hold-minutes", "20", "--confirm-spend", "100"]), h.deps); + const result = await readResult(out); + assert.equal(code, EXIT.fail); + assert.equal(result.stop!.criterion, preflight.criterion); + assert.match(result.stop!.reason, preflight.reason); + assert.ok(h.clock.now() - started < 2 * 60_000, "it does not wait for the start timeout"); + assert.equal(h.fake.issues.size, 1); + const agents = h.fake.requests.filter(request => request.method === "POST" && new URL(request.url).pathname.endsWith("/agents")); + assert.equal(agents.length, 1); + assertCleanedUp(h.fake); + }); + } + + it("passes the preflight when its task starts and finishes between two polls", async () => { + const out = await tempDir(); + const h = harness({ startDelayMs: 0, runDurationMs: 1_000 }); + await main(baseArgs(out, ["--steps", "3"]), h.deps); + const result = await readResult(out); + assert.notEqual(result.stop?.criterion, "run_started"); + const agents = h.fake.requests.filter(request => request.method === "POST" && new URL(request.url).pathname.endsWith("/agents")); + assert.equal(agents.length, 3); + assert.ok(result.steps[0]!.metrics.tasksFinished > 1); + }); + + it("keeps reading runs while it waits for the next step's agents, and stops on a run that fails then", async () => { + const out = await tempDir(); + const h = harness({ failWhenRunningAbove: 0, runDurationMs: 3 * 60_000, agentReadyDelayMs: index => index === 0 ? 3_000 : 10 * 60_000 }); + const started = h.clock.now(); + const code = await main(baseArgs(out, ["--steps", "1,3", "--min-concurrency-ratio", "0.5"]), h.deps); + const result = await readResult(out); + assert.equal(code, EXIT.fail); + assert.equal(result.stop!.stepIndex, 1); + assert.equal(result.stop!.criterion, "failed_runs_zero"); + assert.equal(result.steps[0]!.metrics.runsFailed, 1); + assert.ok(h.clock.now() - started < 6 * 60_000, "the failure is seen during the agent wait, not after it"); + assertCleanedUp(h.fake); + }); + + it("reports a step window equal to its hold when the step takes minutes to prepare", async () => { + const out = await tempDir(); + const h = harness({ agentReadyDelayMs: index => index === 0 ? 3_000 : 5 * 60_000 }); + const started = h.clock.now(); + await main(baseArgs(out, ["--steps", "3", "--drain-minutes", "0"]), h.deps); + const result = await readResult(out); + const metrics = result.steps[0]!.metrics; + assert.ok(metrics.startedAt - started >= 5 * 60_000, "the hold starts after the agents finish setup"); + assert.equal(metrics.endedAt! - metrics.startedAt, 2 * 60_000); + }); + + it("keeps waiting for agent setup through a few failed agent list reads", async () => { + const out = await tempDir(); + const h = harness({ failRequest: (method, route, count) => method === "GET" && route.endsWith("/agents") && count <= 3 + ? { status: 500, body: "{\"error\":\"database busy\"}" } : null }); + await main(baseArgs(out, ["--steps", "2"]), h.deps); + const result = await readResult(out); + assert.notEqual(result.stop?.criterion, "agents_ready"); + assert.equal(result.error, null); + assert.equal(result.steps[0]!.metrics.peakRunning, 2); + }); + + it("stops with the status it saw when the agent list keeps failing during setup", async () => { + const out = await tempDir(); + const h = harness({ failRequest: (method, route, count) => method === "GET" && route.endsWith("/agents") && count <= 5 + ? { status: 500, body: "{\"error\":\"database busy\"}" } : null }); + const code = await main(baseArgs(out, ["--steps", "2"]), h.deps); + const result = await readResult(out); + assert.equal(code, EXIT.fail); + assert.equal(result.error, null); + assert.equal(result.stop!.criterion, "agents_ready"); + assert.match(result.stop!.reason, /could not be read 5 times in a row.*returned 500/); + assertCleanedUp(h.fake); + }); + + it("starts cleanup at once when interrupted while agent list reads fail", async () => { + const out = await tempDir(); + let cleaning = false; + let failed = 0; + const h = harness({ failRequest: (method, route) => { + if (cleaning || method !== "GET" || !route.endsWith("/agents")) return null; + failed += 1; + if (failed === 2) h.interrupt(); + return { status: 500, body: "{\"error\":\"database busy\"}" }; + } }); + const out0 = h.deps.out; + h.deps.out = line => { + if (line.startsWith("Cleaning up")) cleaning = true; + out0(line); + }; + const started = h.clock.now(); + await main(baseArgs(out, ["--steps", "2"]), h.deps); + const result = await readResult(out); + assert.equal(failed, 2, "no agent list read is retried after the interrupt"); + assert.match(result.stop!.reason, /interrupted/); + assert.ok(h.clock.now() - started < 15_000, "cleanup starts within one poll interval"); + assertCleanedUp(h.fake); + }); + it("reports missing optional sources as not measured", async () => { const out = await tempDir(); const h = harness(); diff --git a/scripts/scale-ramp/setup.ts b/scripts/scale-ramp/setup.ts index b442f2c859f..a91fd7dee84 100644 --- a/scripts/scale-ramp/setup.ts +++ b/scripts/scale-ramp/setup.ts @@ -1,4 +1,4 @@ -import { ApiError, type PaperclipClient } from "./client.ts"; +import { ApiError, type Clock, type PaperclipClient } from "./client.ts"; import type { AgentProfile, Options } from "./config.ts"; import { agentName, companyName, mapLimit, projectName, secretPrefix, type RunState, type StateStore } from "./state.ts"; @@ -138,3 +138,92 @@ export async function ensureAgents(input: { } export const AGENT_CREATE_PARALLELISM = 8; +export const LIFECYCLE_ERROR_GRACE_MS = 60_000; +export const AGENT_LIST_FAILURE_LIMIT = 5; + +export class AgentsNotReady extends Error { + constructor(message: string) { + super(message); + this.name = "AgentsNotReady"; + } +} + +const NOT_INVOKABLE = new Set(["paused", "terminated", "pending_approval"]); + +export function agentReady(agent: Json): boolean { + if (agent.lifecycleState === "ready") return !NOT_INVOKABLE.has(String(agent.status)); + return agent.lifecycleState == null && ["idle", "active", "running", "error"].includes(String(agent.status)); +} + +export function describeAgent(agent: Json): string { + const parts = [`agent ${agent.name ?? agent.id} is ${agent.status ?? "missing"}`]; + if (agent.lifecycleState) parts.push(`setup state ${agent.lifecycleState}`); + if (agent.pauseReason) parts.push(`pause reason ${agent.pauseReason}`); + if (Array.isArray(agent.lifecycleHolds) && agent.lifecycleHolds.length) parts.push(`holds ${agent.lifecycleHolds.join(", ")}`); + const reason = agent.lifecycleError ?? agent.errorReason; + return `${parts.join(", ")}; ${reason ? `Paperclip says: ${String(reason).slice(0, 400)}` : "Paperclip gave no reason"}`; +} + +export async function waitForAgentsReady(input: { + api: PaperclipClient; + clock: Clock; + companyId: string; + agentIds: readonly string[]; + timeoutMs: number; + pollMs: number; + shouldStop: () => boolean; + log: (line: string) => void; + tick?: () => Promise; +}): Promise { + const { api, clock } = input; + const started = clock.now(); + const erroredSince = new Map(); + let logged = false; + let readFailures = 0; + for (;;) { + if (input.shouldStop()) return false; + let listed: Json[]; + try { + listed = await api.get(`/api/companies/${input.companyId}/agents`); + readFailures = 0; + } catch (error) { + readFailures += 1; + if (input.shouldStop()) return false; + const message = error instanceof Error ? error.message : String(error); + if (readFailures >= AGENT_LIST_FAILURE_LIMIT) { + throw new AgentsNotReady(`the agent list could not be read ${readFailures} times in a row while waiting for agent setup: ${message.slice(0, 300)}`); + } + if (clock.now() - started >= input.timeoutMs) { + throw new AgentsNotReady(`agent setup could not be checked within ${Math.round(input.timeoutMs / 60_000)} min: ${message.slice(0, 300)}`); + } + await clock.sleep(input.pollMs); + continue; + } + const byId = new Map(listed.map(agent => [String(agent.id), agent])); + const waiting = input.agentIds.map(id => byId.get(id) ?? { id, status: null }).filter(agent => !agentReady(agent)); + if (!waiting.length) { + if (logged) input.log(`All ${input.agentIds.length} agents finished setup after ${Math.round((clock.now() - started) / 1_000)} s`); + return true; + } + if (input.shouldStop() || await input.tick?.()) return false; + const now = clock.now(); + for (const agent of waiting) { + const id = String(agent.id); + if (agent.lifecycleError) { + if (!erroredSince.has(id)) erroredSince.set(id, now); + } else erroredSince.delete(id); + } + const stuck = waiting.find(agent => now - (erroredSince.get(String(agent.id)) ?? now) >= LIFECYCLE_ERROR_GRACE_MS); + if (stuck) { + throw new AgentsNotReady(`${waiting.length} of ${input.agentIds.length} agents cannot finish setup: ${describeAgent(stuck)}`); + } + if (now - started >= input.timeoutMs) { + throw new AgentsNotReady(`${waiting.length} of ${input.agentIds.length} agents did not finish setup within ${Math.round(input.timeoutMs / 60_000)} min: ${describeAgent(waiting[0]!)}`); + } + if (!logged) { + input.log(`Waiting for ${waiting.length} agents to finish setup (${describeAgent(waiting[0]!)})`); + logged = true; + } + await clock.sleep(input.pollMs); + } +}