From 328a954cc163dd3b732aa9d2fed5749dac609078 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sat, 14 Nov 2026 20:10:51 -0600 Subject: [PATCH] test(scaling): add a real-agent ramp harness for hosted instances (#21695) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The control plane must run 1,000 agents at once on one workspace (`doc/plans/2026-11-13-thousand-concurrent-agents.md`) > - Above about 50 agents, the evidence is a load model only. Nobody has run 200 or 1,000 real agents > - Phase 3 of the plan proves the finish line with real runs on hosted instances, in a fixed order > - This pull request adds the harness for those runs. It does not run it against any real instance > - The benefit is that each proof step is one command with a written cost estimate, explicit pass/fail criteria, and a guaranteed cleanup ## Linked Issues or Issue Description **What problem does this solve?** The 1,000-agent plan needs real runs at 50, 200, 500 and 1,000 agents. No tool exists that creates real agents and tasks on a hosted instance, ramps concurrency, measures the control plane, and judges the plan's finish line. **Proposed solution** A script under `scripts/` that ramps real agents through the product API, stops at the first failed criterion, writes a machine-readable result, and always cleans up what it created. **Alternatives considered** Extend the restart-chaos hosted mode. That mode grades release swaps at up to 30 concurrent tasks. The ramp reuses its target guard and its cleanup order, but it has a different workload, different measures and a different stop rule. ## What Changed - `scripts/scale-agents-ramp.ts` and `scripts/scale-ramp/`: the harness. It reads the base URL from a flag or the environment and the credentials from the environment only. It creates or reuses a company, then creates agents, tasks, a project and an optional provider secret. It ramps concurrency in steps (default 10, 50, 100, 200, 500, 1,000) and keeps each agent on one small deterministic task at a time. - It measures per step: tasks started, finished, failed and needing Retry; failed runs; peak running agents; queue wait; time to first token and first activity (from sampled run events); run duration; board GET latency. An optional metrics endpoint supplies event-loop, statement and pool figures. A missing source is reported as "not measured", never as zero. - It evaluates the criteria from the plan's finish line plus harness criteria, writes them into `result.json` and `SUMMARY.md`, and stops at the first failure. - Cleanup runs in a `finally`: pause agents, cancel live runs, archive tasks, terminate and delete agents, delete the stored secret, archive the project and the created company, and wait for sandbox leases. `--cleanup-only ` finishes a cleanup after a crash. `--dry-run` prints the plan and the estimate and makes no request. - Target guard: `--target-kind` is required and must match the host. Shared staging and production hosts need a second explicit flag. The script asks for the host name before it starts unless `--yes` is given. A run above the spend threshold needs `--confirm-spend `. - `scripts/scale-ramp/scale-ramp.test.ts`: 28 tests against an in-process fake API with a virtual clock. They cover ramp logic, stop on failure, criteria evaluation, cleanup after a mid-run crash and an interrupt, `--cleanup-only` with and without a state file, dry run, the production-host guard, the spend gate, and that credentials never appear in output or files. - `scripts/scale-ramp/tsconfig.json` is added to the test programs that CI typechecks. The test file joins the `test:release-registry` node test list, which CI runs. - `doc/plans/2026-11-13-thousand-concurrent-agents.md`: a Phase 3 runbook with commands, cost, criteria, proof order, target prerequisites and open risks. ## Verification - `node scripts/ci-profile/node-test.mjs scripts/scale-ramp/scale-ramp.test.ts scripts/typecheck-test-programs.test.mjs`: 47 tests pass. - `node node_modules/typescript/bin/tsc -p scripts/scale-ramp/tsconfig.json`: no errors. - `node cli/node_modules/tsx/dist/cli.mjs scripts/scale-agents-ramp.ts --base-url https://scale1.test.paperclip.app --target-kind test-stack --steps 50 --hold-minutes 30 --dry-run` prints the plan, the estimate and the criteria, and makes no request. - The harness was not run against any real instance. That needs the owner's go-ahead. ## Risks - Low risk for the product: the change adds a script, tests and docs only. No server code changes. - A real run creates real agents, tasks and sandboxes and spends money. The guard, the typed confirmation, the spend gate, the task cap and the recorded-spend stop limit this. The runbook lists the open risks. - The cost defaults are assumptions until a first real run records actual spend. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, via Claude Code with tool use (shell, file edits). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .gitignore | 1 + .../2026-11-13-thousand-concurrent-agents.md | 136 +++++ package.json | 2 +- scripts/scale-agents-ramp.ts | 3 + scripts/scale-ramp/cleanup.ts | 180 ++++++ scripts/scale-ramp/client.ts | 164 ++++++ scripts/scale-ramp/config.ts | 202 +++++++ scripts/scale-ramp/criteria.ts | 160 ++++++ scripts/scale-ramp/estimate.ts | 56 ++ scripts/scale-ramp/fake-paperclip.ts | 308 ++++++++++ scripts/scale-ramp/guard.ts | 70 +++ scripts/scale-ramp/main.ts | 180 ++++++ scripts/scale-ramp/ramp.ts | 474 +++++++++++++++ scripts/scale-ramp/report.ts | 164 ++++++ scripts/scale-ramp/scale-ramp.test.ts | 544 ++++++++++++++++++ scripts/scale-ramp/setup.ts | 140 +++++ scripts/scale-ramp/state.ts | 93 +++ scripts/scale-ramp/stats.ts | 26 + scripts/scale-ramp/tsconfig.json | 13 + scripts/scale-ramp/workload.ts | 69 +++ scripts/typecheck-test-programs.mjs | 1 + .../runner-e2e/source-sdk-public-inputs.json | 2 +- 22 files changed, 2986 insertions(+), 2 deletions(-) create mode 100644 scripts/scale-agents-ramp.ts create mode 100644 scripts/scale-ramp/cleanup.ts create mode 100644 scripts/scale-ramp/client.ts create mode 100644 scripts/scale-ramp/config.ts create mode 100644 scripts/scale-ramp/criteria.ts create mode 100644 scripts/scale-ramp/estimate.ts create mode 100644 scripts/scale-ramp/fake-paperclip.ts create mode 100644 scripts/scale-ramp/guard.ts create mode 100644 scripts/scale-ramp/main.ts create mode 100644 scripts/scale-ramp/ramp.ts create mode 100644 scripts/scale-ramp/report.ts create mode 100644 scripts/scale-ramp/scale-ramp.test.ts create mode 100644 scripts/scale-ramp/setup.ts create mode 100644 scripts/scale-ramp/state.ts create mode 100644 scripts/scale-ramp/stats.ts create mode 100644 scripts/scale-ramp/tsconfig.json create mode 100644 scripts/scale-ramp/workload.ts diff --git a/.gitignore b/.gitignore index d0b463a8b6e..e985a535a12 100644 --- a/.gitignore +++ b/.gitignore @@ -104,3 +104,4 @@ test-results/ # Content-addressed CI task results .turbo/ +scripts/scale-ramp/results/ diff --git a/doc/plans/2026-11-13-thousand-concurrent-agents.md b/doc/plans/2026-11-13-thousand-concurrent-agents.md index c9c447a53e4..852a9fdf72a 100644 --- a/doc/plans/2026-11-13-thousand-concurrent-agents.md +++ b/doc/plans/2026-11-13-thousand-concurrent-agents.md @@ -358,6 +358,139 @@ of other servers keep their records. 3. `paperclip.staging` only after the owner's explicit go-ahead. 4. Repeat the 1,000 step with real model calls for a short window. +### Phase 3 runbook: real-agent ramp + +`scripts/scale-agents-ramp.ts` runs real agents on a hosted instance through +the product API only. It ramps concurrency in steps, holds each step, stops at +the first failed criterion, and always cleans up. It has run only against the +in-process fake API of its tests (`scripts/scale-ramp/scale-ramp.test.ts`). +A run on a real instance needs the owner's go-ahead. + +**What one run does.** It creates a company named `Scale ramp `, or +uses `--company ` (a Cloud-managed instance refuses company creation, so +pass the company there). It stores the provider key as a company secret, or +reuses `PAPERCLIP_SCALE_PROVIDER_SECRET_ID`. It creates one project, and one +agent per concurrent slot (`maxConcurrentRuns` 1, heartbeat off, warm runner, +the company's single sandbox environment). Each agent works one task at a +time. When a task ends, the agent gets a new one, so the step keeps its +concurrency. Each task is the same small job: stream a sentence, run `pwd` +and `ls`, write `scale-.txt` in the workspace, read it back, post a +`CHECKPOINT` comment, post a `DONE` comment and mark the task done. + +**Run it.** + +```sh +export PAPERCLIP_SCALE_TOKEN= # never a flag +export OPENROUTER_API_KEY= # or PAPERCLIP_SCALE_PROVIDER_SECRET_ID +H=.test.paperclip.app +S="node cli/node_modules/tsx/dist/cli.mjs scripts/scale-agents-ramp.ts --base-url https://$H --target-kind test-stack" +$S --steps 10,50 --hold-minutes 20 --dry-run # plan and estimate; no request +$S --steps 10,50 --hold-minutes 20 --confirm-spend 15 # first real 50-agent run +$S --cleanup-only # finish a cleanup after a crash +``` + +The script asks you to type the host name before it creates anything, unless +`--yes` is given. Optional environment: `PAPERCLIP_SCALE_COMPANY_ID`, +`PAPERCLIP_SCALE_ENVIRONMENT_ID`, `PAPERCLIP_SCALE_PROVIDER`, +`PAPERCLIP_SCALE_MODEL` (default `openrouter/deepseek/deepseek-v4-flash-0731`, +the qualified OpenCode model that restart-chaos hosted runs also use), +`PAPERCLIP_SCALE_ADAPTER_CONFIG` (JSON merged into the agent's adapter config, +for example to use a scripted runner when one exists), and +`PAPERCLIP_SCALE_METRICS_URL` with `PAPERCLIP_SCALE_METRICS_TOKEN`. Results go +to `scripts/scale-ramp/results//`: `result.json`, `SUMMARY.md` and +`state.json`. Exit codes: 0 pass, 1 fail, 2 refused, 3 incomplete, 4 error or +interrupted. + +**Target guard.** `--target-kind` is required and must match the host: +`local` (localhost), `test-stack` (`.staging.paperclip.app` or +`.test.paperclip.app`, the restart-chaos hosted pattern), +`shared-staging` (`paperclip.staging.paperclip.app`, which also needs +`--allow-shared-staging`), or `production` (any other host, which also needs +`--allow-production-host `). Every request is checked against +the one origin. + +**Cost.** The script prints an estimate before it starts and refuses a run +above `--spend-threshold-usd` (default $5) unless `--confirm-spend ` +is at least the estimate. It never creates more tasks than the printed cap, +and it stops when the model spend the runs record passes the confirmed +amount. The defaults are assumptions: $0.02 model spend per task and $0.10 +per sandbox-hour (`PAPERCLIP_SCALE_MODEL_USD_PER_TASK`, +`PAPERCLIP_SCALE_SANDBOX_USD_PER_HOUR`). Correct them from the recorded spend +of the first run. + +| Steps | Hold | Tasks (cap) | Estimate | +| --- | --- | --- | --- | +| 10, 50 | 20 min | 420 (525) | $13 | +| 200 | 60 min | 4,000 (5,000) | $122 | +| 1,000 | 60 min | 20,000 (25,000) | $610 | + +**Pass.** The criteria are written into every result. A step passes when no +measured criterion fails. The run passes when every step passes and every +criterion was measured; it is `incomplete` when a criterion was not measured. +A missing source is reported as "not measured", never as zero. + +| Criterion | Source | Target | Measured from | +| --- | --- | --- | --- | +| Runs that need Retry | finish line | 0, stops at once | task view offers Retry | +| Failed or timed-out runs | harness | 0, stops at once | run list | +| Tasks at rest but not done | harness | 0 | task status | +| Peak running agents | harness | at least 90% of the step | run list | +| Board page API requests, p90 | finish line | below 500 ms | a GET mix every 10 s: dashboard, issues, agents, live runs, sidebar badges | +| Board API failures | harness | 0 | the same requests | +| Queue wait before start, p95 | harness | at most 60 s | run created to started | +| First visible activity, p95 | harness | at most 60 s | run events of up to 50 runs per step | +| Main-thread CPU or event-loop utilization | finish line | below 0.6 | metrics endpoint | +| Run event lateness, p99 | finish line | below 1 s | metrics endpoint | +| Agent API time before the handler, p99 | finish line | below 100 ms | metrics endpoint | +| Lease renewals that miss their deadline | finish line | 0 | metrics endpoint | +| Kill one server process | finish line | runs resume, no Retry | not measured by this harness | + +The server has no metrics endpoint today. `PAPERCLIP_SCALE_METRICS_URL` must +answer a JSON object with any of `mainThreadCpu`, `eventLoopUtilization` +(0 to 1, highest server process), `eventLoopDelayP99Ms`, +`statementsPerSecond`, `poolWaitP99Ms`, `runEventLatenessP99Ms`, +`agentApiPreHandlerP99Ms` and `leaseRenewalsMissed` (a counter). Without it, +the four server criteria are not measured, and a run can be at best +`incomplete`. The harness does not read the database. + +**Proof order.** A hosted stack that is not `paperclip.staging` first +(`--target-kind test-stack`): 10 and 50, then 200, then 500 and 1,000, one +hour per step from 200 on. `paperclip.staging` only after the owner's +explicit go-ahead. Production last. + +**Check the target first.** + +- Sandbox quota and concurrency of the provider account for the step size + (50, 200, 1,000 sandboxes at once), and its start rate. +- Provider rate limits (OpenRouter requests and tokens per minute) for the + step size. +- Database connection budget: each server process opens its main, + checkpoint and liveness pools plus dedicated connections (about 10 to 20 + at 1,000 runs, see change 1g), and a burst of starts can open more. +- Company budget and hard stop: the company must allow the run's spend, or + the budget stop pauses it part way. +- Nothing else important runs on the instance; the run creates real tasks. + +**Risks to know before approving a real run.** + +- Sandbox cost and quota: the estimate counts sandbox-hours at an assumed + rate. 200 agents for an hour is about 220 sandbox-hours, and 1,000 is about + 1,100. +- Model spend: about $0.02 per task is an assumption; 1,000 agents create up + to 25,000 tasks in an hour. +- The run creates real companies, agents, tasks, comments, runs, cost rows + and sandboxes on the instance. Cleanup archives the tasks, project and + created company, deletes agents (an agent with cost history stays + terminated) and the stored secret. It cannot remove run history or cost + rows. +- Cleanup cancels live runs, terminates agents and waits up to 5 minutes for + agents to terminate and up to 5 minutes for the run's sandbox leases to be + released. At 1,000 agents expect 10 to 20 minutes, mostly requests at 8 in + parallel. If the harness dies, `--cleanup-only ` finishes from + `state.json`, or by name when the file is gone. +- At 1,000 agents the harness itself adds load: one run list read, ten task + reads and a board GET mix per poll (15 s), plus task creation. + ## Decisions for the owner | # | Decision | Recommendation | @@ -376,3 +509,6 @@ of other servers keep their records. - Staging's pool size and run-log storage mode were not read. - Tool-gateway calls, recovery sweeps, and browser fan-out are not in the model. +- The real-agent ramp (`scripts/scale-agents-ramp.ts`) has run only against + the fake API of its tests. Its cost defaults are assumptions, and the + server criteria need a metrics endpoint that does not exist yet. diff --git a/package.json b/package.json index 590fd8f5550..2d2a4abebb6 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "smoke:notion-generic-live": "node scripts/smoke/notion-generic-live.mjs", "smoke:posthog-live": "node scripts/smoke/posthog-live.mjs", "smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs", - "test:release-registry": "node scripts/ci-profile/node-test.mjs scripts/ci-profile/reporters.test.mjs scripts/ci/retry-reporter.test.mjs scripts/__tests__/runner-vitest-package-lanes.test.mjs scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/public-install-closure.test.mjs scripts/__tests__/public-install-registry.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/__tests__/runner-release-crate-cache.test.mjs scripts/__tests__/runner-debug-crate-cache.test.mjs scripts/__tests__/agent-build-cache.test.mjs scripts/__tests__/agent-preflight.test.mjs scripts/__tests__/vitest-worker-isolation.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/draft-stable-notes.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs scripts/typecheck-test-programs.test.mjs scripts/__tests__/local-verify.test.mjs scripts/test-sandbox-providers.test.mjs scripts/__tests__/ci-typecheck-lane.test.mjs", + "test:release-registry": "node scripts/ci-profile/node-test.mjs scripts/ci-profile/reporters.test.mjs scripts/ci/retry-reporter.test.mjs scripts/__tests__/runner-vitest-package-lanes.test.mjs scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/public-install-closure.test.mjs scripts/__tests__/public-install-registry.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/__tests__/runner-release-crate-cache.test.mjs scripts/__tests__/runner-debug-crate-cache.test.mjs scripts/__tests__/agent-build-cache.test.mjs scripts/__tests__/agent-preflight.test.mjs scripts/__tests__/vitest-worker-isolation.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/draft-stable-notes.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs scripts/typecheck-test-programs.test.mjs scripts/__tests__/local-verify.test.mjs scripts/test-sandbox-providers.test.mjs scripts/__tests__/ci-typecheck-lane.test.mjs scripts/scale-ramp/scale-ramp.test.ts", "storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs", "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts", "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack", diff --git a/scripts/scale-agents-ramp.ts b/scripts/scale-agents-ramp.ts new file mode 100644 index 00000000000..13c9f8ebaf0 --- /dev/null +++ b/scripts/scale-agents-ramp.ts @@ -0,0 +1,3 @@ +import { main, processDeps } from "./scale-ramp/main.ts"; + +process.exitCode = await main(process.argv.slice(2), processDeps()); diff --git a/scripts/scale-ramp/cleanup.ts b/scripts/scale-ramp/cleanup.ts new file mode 100644 index 00000000000..f40b169ca97 --- /dev/null +++ b/scripts/scale-ramp/cleanup.ts @@ -0,0 +1,180 @@ +import { ApiError, type Clock, type PaperclipClient } from "./client.ts"; +import { agentPrefix, companyName, mapLimit, projectName, secretPrefix, type RunState } from "./state.ts"; + +type Json = Record; + +export interface CleanupReport { + agentsPaused: number; + runsCancelled: number; + tasksArchived: number; + agentsTerminated: number; + agentsDeleted: number; + agentsKeptForCostHistory: number; + secretDeleted: boolean; + projectArchived: boolean; + companyArchived: boolean; + activeSandboxLeases: number | null; + durationMs: number; + errors: string[]; +} + +export const LIVE_RUN_STATUSES = new Set(["queued", "running", "scheduled_retry"]); +const PARALLELISM = 8; + +export interface CleanupInput { + api: PaperclipClient; + clock: Clock; + label: string; + state: RunState; + terminateWaitMs?: number; + leaseWaitMs?: number; + pollMs?: number; + log?: (line: string) => void; +} + +export async function discoverState(api: PaperclipClient, label: string, companyId: string | null, baseUrl: string, now: number): Promise { + let resolved = companyId; + let created = false; + if (!resolved) { + const companies = await api.get("/api/companies"); + const match = companies.find(company => company.name === companyName(label)); + if (!match) throw new Error(`No state file for ${label} and no company named "${companyName(label)}"; pass --company `); + resolved = String(match.id); + created = true; + } + return { + schema: "paperclip.scale-ramp.state/v1", label, baseUrl, companyId: resolved, companyCreated: created, environmentId: null, + projectId: null, secretId: null, agentIds: [], taskIds: [], createdAt: new Date(now).toISOString(), + }; +} + +export async function cleanupRun(input: CleanupInput): Promise { + const { api, clock, label, state } = input; + const started = clock.now(); + const pollMs = input.pollMs ?? 3_000; + const report: CleanupReport = { + agentsPaused: 0, runsCancelled: 0, tasksArchived: 0, agentsTerminated: 0, agentsDeleted: 0, agentsKeptForCostHistory: 0, + secretDeleted: false, projectArchived: false, companyArchived: false, activeSandboxLeases: null, durationMs: 0, errors: [], + }; + const attempt = async (what: string, action: () => Promise): Promise => { + try { + await action(); + return true; + } catch (error) { + report.errors.push(`${what}: ${error instanceof Error ? error.message : String(error)}`.slice(0, 300)); + return false; + } + }; + const read = async (what: string, route: string, fallback: T): Promise => { + try { + return await api.get(route); + } catch (error) { + report.errors.push(`${what}: ${error instanceof Error ? error.message : String(error)}`.slice(0, 300)); + return fallback; + } + }; + const companyId = state.companyId; + if (!companyId) { + report.durationMs = clock.now() - started; + return report; + } + const prefix = agentPrefix(label); + const listed = await read("list agents", `/api/companies/${companyId}/agents`, []); + const agents = listed.filter(agent => String(agent.name ?? "").startsWith(prefix)); + const agentIds = new Set(agents.map(agent => String(agent.id))); + input.log?.(`Cleanup ${label}: ${agents.length} agents, ${state.taskIds.length} recorded tasks`); + + await mapLimit(agents.filter(agent => agent.status !== "terminated" && agent.status !== "paused"), PARALLELISM, async agent => { + if (await attempt(`pause agent ${agent.id}`, () => api.patch(`/api/agents/${agent.id}`, { status: "paused" }))) report.agentsPaused += 1; + }); + + const runs = await read("list runs", `/api/companies/${companyId}/heartbeat-runs?summary=true&limit=1000`, []); + const live = new Map(); + for (const run of runs) if (agentIds.has(String(run.agentId)) && LIVE_RUN_STATUSES.has(String(run.status))) live.set(String(run.id), run); + if (runs.length >= 1_000) { + await mapLimit([...agentIds], PARALLELISM, async agentId => { + const own = await read(`list runs of ${agentId}`, `/api/companies/${companyId}/heartbeat-runs?agentId=${agentId}&summary=true&limit=50`, []); + for (const run of own) if (LIVE_RUN_STATUSES.has(String(run.status))) live.set(String(run.id), run); + }); + } + await mapLimit([...live.keys()], PARALLELISM, async runId => { + if (await attempt(`cancel run ${runId}`, () => api.post(`/api/heartbeat-runs/${runId}/cancel`))) report.runsCancelled += 1; + }); + + const taskIds = new Set(state.taskIds); + await mapLimit([...agentIds], PARALLELISM, async agentId => { + const issues = await read(`list tasks of ${agentId}`, `/api/companies/${companyId}/issues?assigneeAgentId=${agentId}&limit=500`, []); + for (const issue of issues) if (issue.assigneeAgentId === agentId && !issue.hiddenAt) taskIds.add(String(issue.id)); + }); + await mapLimit([...taskIds], PARALLELISM, async taskId => { + const archived = await attempt(`archive task ${taskId}`, async () => { + try { + await api.patch(`/api/issues/${taskId}`, { hiddenAt: new Date(clock.now()).toISOString() }); + } catch (error) { + if (!(error instanceof ApiError && error.status === 404)) throw error; + } + }); + if (archived) report.tasksArchived += 1; + }); + + await mapLimit(agents.filter(agent => agent.status !== "terminated"), PARALLELISM, async agent => { + if (await attempt(`terminate agent ${agent.id}`, () => api.post(`/api/agents/${agent.id}/terminate`))) report.agentsTerminated += 1; + }); + const pending = new Set(agentIds); + for (const until = clock.now() + (input.terminateWaitMs ?? 300_000); pending.size && clock.now() < until;) { + const current = await read("list agents", `/api/companies/${companyId}/agents`, []); + for (const agent of current) { + if (!pending.has(String(agent.id))) continue; + if (agent.status === "terminated" && (agent.lifecycleState == null || agent.lifecycleState === "terminated")) pending.delete(String(agent.id)); + } + for (const id of [...pending]) if (!current.some(agent => String(agent.id) === id)) pending.delete(id); + if (pending.size) await clock.sleep(pollMs); + } + if (pending.size) report.errors.push(`${pending.size} agents were not terminated in time; run --cleanup-only ${label} again`); + await mapLimit([...agentIds].filter(id => !pending.has(id)), PARALLELISM, async agentId => { + try { + await api.delete(`/api/agents/${agentId}`); + report.agentsDeleted += 1; + } catch (error) { + if (error instanceof ApiError && error.status === 409 && error.message.includes("agent_cost_history_retained")) report.agentsKeptForCostHistory += 1; + else report.errors.push(`delete agent ${agentId}: ${error instanceof Error ? error.message : String(error)}`.slice(0, 300)); + } + }); + + const secrets = await read("list secrets", `/api/companies/${companyId}/secrets`, []); + for (const secret of secrets.filter(secret => String(secret.name ?? "").startsWith(secretPrefix(label)))) { + if (await attempt(`delete secret ${secret.id}`, () => api.delete(`/api/secrets/${secret.id}`))) report.secretDeleted = true; + } + + const projects = await read("list projects", `/api/companies/${companyId}/projects`, []); + for (const project of projects.filter(project => project.name === projectName(label) && !project.archivedAt)) { + if (await attempt(`archive project ${project.id}`, () => api.patch(`/api/projects/${project.id}`, { archivedAt: new Date(clock.now()).toISOString() }))) { + report.projectArchived = true; + } + } + + if (state.environmentId && taskIds.size) { + for (const until = clock.now() + (input.leaseWaitMs ?? 300_000); ;) { + let leases: Json[] | null = null; + try { + leases = await api.get(`/api/environments/${state.environmentId}/leases?status=active`); + } catch { + leases = null; + } + if (leases === null) break; + report.activeSandboxLeases = leases.filter(lease => taskIds.has(String(lease.issueId))).length; + if (report.activeSandboxLeases === 0 || clock.now() >= until) break; + await clock.sleep(pollMs); + } + if (report.activeSandboxLeases) report.errors.push(`${report.activeSandboxLeases} sandbox leases of this run are still active`); + } + + if (state.companyCreated) { + const company = await read("read company", `/api/companies/${companyId}`, null); + if (company && company.name === companyName(label) && company.status !== "archived") { + if (await attempt(`archive company ${companyId}`, () => api.post(`/api/companies/${companyId}/archive`))) report.companyArchived = true; + } + } + report.durationMs = clock.now() - started; + return report; +} diff --git a/scripts/scale-ramp/client.ts b/scripts/scale-ramp/client.ts new file mode 100644 index 00000000000..d0272f4d756 --- /dev/null +++ b/scripts/scale-ramp/client.ts @@ -0,0 +1,164 @@ +import { assertSameOrigin, type Target } from "./guard.ts"; + +export type Fetcher = (url: string, init: RequestInit) => Promise; + +export interface Clock { + now(): number; + sleep(ms: number): Promise; +} + +export const realClock: Clock = { + now: () => Date.now(), + sleep: ms => new Promise(resolve => setTimeout(resolve, ms)), +}; + +export interface RequestRecord { + at: number; + method: string; + path: string; + status: number | null; + ms: number; + ok: boolean; + kind: "board" | "harness"; +} + +export class ApiError extends Error { + readonly status: number | null; + readonly code: string | null; + + constructor(status: number | null, message: string, code: string | null = null) { + super(message); + this.name = "ApiError"; + this.status = status; + this.code = code; + } +} + +export class Redactor { + private readonly values: string[]; + + constructor(values: ReadonlyArray) { + this.values = values.filter((value): value is string => typeof value === "string" && value.length >= 4) + .sort((a, b) => b.length - a.length); + } + + text(input: string): string { + let out = input; + for (const value of this.values) out = out.split(value).join("[redacted]"); + return out.replace(/(bearer\s+)[A-Za-z0-9._~+/=-]{8,}/gi, "$1[redacted]"); + } + + json(value: T): T { + return JSON.parse(this.text(JSON.stringify(value))) as T; + } +} + +const UUID = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi; +const TRANSIENT = new Set([502, 503, 504]); + +export function pathClass(route: string): string { + return route.split("?")[0]!.replace(UUID, ":id"); +} + +export class PaperclipClient { + readonly records: RequestRecord[] = []; + private readonly target: Target; + private readonly token: string; + private readonly fetcher: Fetcher; + private readonly clock: Clock; + private readonly redactor: Redactor; + private readonly timeoutMs: number; + + constructor(target: Target, token: string, fetcher: Fetcher, clock: Clock, redactor: Redactor, timeoutMs = 30_000) { + this.target = target; + this.token = token; + this.fetcher = fetcher; + this.clock = clock; + this.redactor = redactor; + this.timeoutMs = timeoutMs; + } + + private async once(method: string, route: string, body: unknown, kind: RequestRecord["kind"]) { + const url = `${this.target.baseUrl}${route}`; + assertSameOrigin(this.target, url); + const started = this.clock.now(); + try { + const response = await this.fetcher(url, { + method, + cache: "no-store", + signal: AbortSignal.timeout(this.timeoutMs), + headers: { + authorization: `Bearer ${this.token}`, + accept: "application/json", + origin: this.target.baseUrl, + ...(body === undefined ? {} : { "content-type": "application/json" }), + }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + const text = await response.text(); + const ms = this.clock.now() - started; + const answered = response.ok || (kind === "harness" && [400, 404, 409, 422].includes(response.status)); + this.records.push({ at: started, method, path: pathClass(route), status: response.status, ms, ok: answered, kind }); + return { status: response.status, ok: response.ok, text }; + } catch (error) { + this.records.push({ at: started, method, path: pathClass(route), status: null, ms: this.clock.now() - started, ok: false, kind }); + return { status: null, ok: false, text: error instanceof Error ? error.message : String(error) }; + } + } + + async request(method: string, route: string, body?: unknown, kind: RequestRecord["kind"] = "harness"): Promise { + for (let attempt = 1; ; attempt += 1) { + const result = await this.once(method, route, body, kind); + if (result.ok) return (result.text ? JSON.parse(result.text) : null) as T; + const transient = result.status === null || TRANSIENT.has(result.status) || result.status === 429; + if (!transient || (method !== "GET" && result.status !== 429) || attempt >= 3) { + let code: string | null = null; + try { + const parsed = JSON.parse(result.text) as { code?: unknown; details?: { code?: unknown } }; + code = typeof parsed.code === "string" ? parsed.code : typeof parsed.details?.code === "string" ? parsed.details.code : null; + } catch { + code = null; + } + const detail = this.redactor.text(result.text.slice(0, 300)); + throw new ApiError(result.status, `${method} ${pathClass(route)} ${result.status === null ? "got no answer" : `returned ${result.status}`}: ${detail}`, code); + } + await this.clock.sleep(1_000 * attempt); + } + } + + get(route: string, kind: RequestRecord["kind"] = "harness") { + return this.request("GET", route, undefined, kind); + } + + post(route: string, body: unknown = {}) { + return this.request("POST", route, body); + } + + patch(route: string, body: unknown = {}) { + return this.request("PATCH", route, body); + } + + async delete(route: string, allowNotFound = true): Promise { + try { + await this.request("DELETE", route); + } catch (error) { + if (!(allowNotFound && error instanceof ApiError && error.status === 404)) throw error; + } + } +} + +export async function readMetrics(fetcher: Fetcher, url: string, token: string | null, clock: Clock): Promise | null> { + try { + const response = await fetcher(url, { + method: "GET", + cache: "no-store", + signal: AbortSignal.timeout(10_000), + headers: { accept: "application/json", ...(token ? { authorization: `Bearer ${token}` } : {}) }, + }); + if (!response.ok) return null; + const body = await response.json() as unknown; + return body && typeof body === "object" && !Array.isArray(body) ? { ...(body as Record), sampledAt: clock.now() } : null; + } catch { + return null; + } +} diff --git a/scripts/scale-ramp/config.ts b/scripts/scale-ramp/config.ts new file mode 100644 index 00000000000..769331262d2 --- /dev/null +++ b/scripts/scale-ramp/config.ts @@ -0,0 +1,202 @@ +import path from "node:path"; +import { parseArgs } from "node:util"; +import { resolveTarget, type Target } from "./guard.ts"; + +export const DEFAULT_STEPS = [10, 50, 100, 200, 500, 1_000]; +export const QUALIFIED_OPENCODE_MODEL = "openrouter/deepseek/deepseek-v4-flash-0731"; +export const TOKEN_ENV = "PAPERCLIP_SCALE_TOKEN"; +export const FALLBACK_TOKEN_ENV = "PAPERCLIP_HOSTED_TOKEN"; +export const RESULTS_ROOT = path.join(import.meta.dirname, "results"); + +export type Mode = "run" | "dry-run" | "cleanup"; + +export interface Step { + index: number; + concurrency: number; + holdMs: number; +} + +export interface Thresholds { + boardP90Ms: number; + queueWaitP95Ms: number; + firstActivityP95Ms: number; + minConcurrencyRatio: number; + maxEventLoopUtilization: number; + runEventLatenessP99Ms: number; + agentApiPreHandlerP99Ms: number; +} + +export interface Pricing { + modelUsdPerTask: number; + sandboxUsdPerHour: number; +} + +export interface AgentProfile { + provider: string; + model: string; + credentialKey: string | null; + providerSecretId: string | null; + adapterOverrides: Record; +} + +export interface Options { + mode: Mode; + target: Target; + label: string; + outDir: string; + steps: Step[]; + drainMs: number; + pollMs: number; + boardProbeMs: number; + taskMinutes: number; + maxTasks: number | null; + companyId: string | null; + environmentId: string | null; + yes: boolean; + confirmSpendUsd: number | null; + spendThresholdUsd: number; + eventSamplePerStep: number; + thresholds: Thresholds; + pricing: Pricing; + profile: AgentProfile; + metricsUrl: string | null; +} + +export interface Secrets { + token: string | null; + credential: string | null; + metricsToken: string | null; +} + +const LABEL = /^scale-[a-z0-9-]{3,40}$/; + +function number(raw: string | undefined, name: string, fallback: number, min: number, max: number): number { + if (raw === undefined || raw === "") return fallback; + const value = Number(raw); + if (!Number.isFinite(value) || value < min || value > max) throw new Error(`${name} must be a number from ${min} to ${max}`); + return value; +} + +export function parseSteps(raw: string, holdMs: number): Step[] { + const values = raw.split(",").map(part => part.trim()).filter(Boolean).map(part => { + if (!/^[0-9]+$/.test(part)) throw new Error(`--steps takes whole numbers separated by commas, not "${part}"`); + return Number(part); + }); + if (!values.length) throw new Error("--steps needs at least one step"); + values.forEach((value, index) => { + if (value < 1 || value > 5_000) throw new Error("each step must be from 1 to 5000 agents"); + if (index > 0 && value <= values[index - 1]!) throw new Error("--steps must increase"); + }); + return values.map((concurrency, index) => ({ index, concurrency, holdMs })); +} + +export function defaultLabel(now: number, random: () => number = Math.random): string { + const stamp = new Date(now).toISOString().slice(0, 16).replace(/[-:T]/g, "").toLowerCase(); + return `scale-${stamp}-${Math.floor(random() * 0xffff).toString(16).padStart(4, "0")}`; +} + +export function readSecrets(env: NodeJS.ProcessEnv, credentialKey: string | null): Secrets { + const pick = (value: string | undefined) => value?.trim() ? value.trim() : null; + return { + token: pick(env[TOKEN_ENV]) ?? pick(env[FALLBACK_TOKEN_ENV]), + credential: credentialKey ? pick(env[credentialKey]) : null, + metricsToken: pick(env.PAPERCLIP_SCALE_METRICS_TOKEN), + }; +} + +function adapterOverrides(raw: string | undefined): Record { + if (!raw) return {}; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw new Error("PAPERCLIP_SCALE_ADAPTER_CONFIG must be a JSON object"); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("PAPERCLIP_SCALE_ADAPTER_CONFIG must be a JSON object"); + return parsed as Record; +} + +export function parseOptions(argv: readonly string[], env: NodeJS.ProcessEnv, now: number, random?: () => number): Options { + const { values } = parseArgs({ + args: argv[0] === "--" ? argv.slice(1) : [...argv], + strict: true, + options: { + "base-url": { type: "string" }, + "target-kind": { type: "string" }, + "allow-shared-staging": { type: "boolean" }, + "allow-production-host": { type: "string" }, + steps: { type: "string" }, + "hold-minutes": { type: "string" }, + "drain-minutes": { type: "string" }, + "poll-seconds": { type: "string" }, + "board-probe-seconds": { type: "string" }, + "task-minutes": { type: "string" }, + "max-tasks": { type: "string" }, + company: { type: "string" }, + environment: { type: "string" }, + label: { type: "string" }, + out: { type: "string" }, + "dry-run": { type: "boolean" }, + "cleanup-only": { type: "string" }, + yes: { type: "boolean" }, + "confirm-spend": { type: "string" }, + "spend-threshold-usd": { type: "string" }, + "event-sample": { type: "string" }, + "max-queue-wait-p95-seconds": { type: "string" }, + "max-first-activity-p95-seconds": { type: "string" }, + "min-concurrency-ratio": { type: "string" }, + }, + }); + if (values["dry-run"] && values["cleanup-only"]) throw new Error("--dry-run and --cleanup-only exclude each other"); + const mode: Mode = values["cleanup-only"] ? "cleanup" : values["dry-run"] ? "dry-run" : "run"; + const target = resolveTarget({ + baseUrl: values["base-url"] ?? env.PAPERCLIP_SCALE_BASE_URL, + kind: values["target-kind"], + allowSharedStaging: values["allow-shared-staging"] === true, + allowProductionHost: values["allow-production-host"], + }); + const label = values["cleanup-only"] ?? values.label ?? defaultLabel(now, random); + if (!LABEL.test(label)) throw new Error("The run id must look like scale-<3 to 40 lowercase letters, digits or dashes>"); + const holdMs = number(values["hold-minutes"], "--hold-minutes", 10, 1, 240) * 60_000; + const credentialKey = env.PAPERCLIP_SCALE_CREDENTIAL_KEY === "" ? null : env.PAPERCLIP_SCALE_CREDENTIAL_KEY ?? "OPENROUTER_API_KEY"; + const confirmSpend = values["confirm-spend"]; + return { + mode, + target, + label, + outDir: path.resolve(values.out ?? env.PAPERCLIP_SCALE_RESULTS_DIR ?? RESULTS_ROOT, label), + steps: parseSteps(values.steps ?? DEFAULT_STEPS.join(","), holdMs), + drainMs: number(values["drain-minutes"], "--drain-minutes", 15, 0, 240) * 60_000, + pollMs: number(values["poll-seconds"], "--poll-seconds", 15, 1, 300) * 1_000, + boardProbeMs: number(values["board-probe-seconds"], "--board-probe-seconds", 10, 1, 300) * 1_000, + taskMinutes: number(values["task-minutes"], "--task-minutes", 3, 0.5, 120), + maxTasks: values["max-tasks"] === undefined ? null : number(values["max-tasks"], "--max-tasks", 0, 1, 1_000_000), + companyId: values.company ?? env.PAPERCLIP_SCALE_COMPANY_ID ?? null, + environmentId: values.environment ?? env.PAPERCLIP_SCALE_ENVIRONMENT_ID ?? null, + yes: values.yes === true, + confirmSpendUsd: confirmSpend === undefined ? null : number(confirmSpend, "--confirm-spend", 0, 0, 1_000_000), + spendThresholdUsd: number(values["spend-threshold-usd"], "--spend-threshold-usd", 5, 0, 1_000_000), + eventSamplePerStep: number(values["event-sample"], "--event-sample", 50, 0, 10_000), + thresholds: { + boardP90Ms: 500, + queueWaitP95Ms: number(values["max-queue-wait-p95-seconds"], "--max-queue-wait-p95-seconds", 60, 1, 3_600) * 1_000, + firstActivityP95Ms: number(values["max-first-activity-p95-seconds"], "--max-first-activity-p95-seconds", 60, 1, 3_600) * 1_000, + minConcurrencyRatio: number(values["min-concurrency-ratio"], "--min-concurrency-ratio", 0.9, 0.1, 1), + maxEventLoopUtilization: 0.6, + runEventLatenessP99Ms: 1_000, + agentApiPreHandlerP99Ms: 100, + }, + pricing: { + modelUsdPerTask: number(env.PAPERCLIP_SCALE_MODEL_USD_PER_TASK, "PAPERCLIP_SCALE_MODEL_USD_PER_TASK", 0.02, 0, 100), + sandboxUsdPerHour: number(env.PAPERCLIP_SCALE_SANDBOX_USD_PER_HOUR, "PAPERCLIP_SCALE_SANDBOX_USD_PER_HOUR", 0.1, 0, 100), + }, + profile: { + provider: env.PAPERCLIP_SCALE_PROVIDER ?? "opencode", + model: env.PAPERCLIP_SCALE_MODEL ?? QUALIFIED_OPENCODE_MODEL, + credentialKey, + providerSecretId: env.PAPERCLIP_SCALE_PROVIDER_SECRET_ID ?? null, + adapterOverrides: adapterOverrides(env.PAPERCLIP_SCALE_ADAPTER_CONFIG), + }, + metricsUrl: env.PAPERCLIP_SCALE_METRICS_URL ?? null, + }; +} diff --git a/scripts/scale-ramp/criteria.ts b/scripts/scale-ramp/criteria.ts new file mode 100644 index 00000000000..be596ff392e --- /dev/null +++ b/scripts/scale-ramp/criteria.ts @@ -0,0 +1,160 @@ +import type { Step, Thresholds } from "./config.ts"; +import type { Distribution } from "./stats.ts"; + +export interface ServerMetrics { + mainThreadCpuMax: number | null; + eventLoopUtilizationMax: number | null; + eventLoopDelayP99MsMax: number | null; + statementsPerSecondMax: number | null; + poolWaitP99MsMax: number | null; + runEventLatenessP99MsMax: number | null; + agentApiPreHandlerP99MsMax: number | null; + leaseRenewalsMissed: number | null; + samples: number; +} + +export interface StepMetrics { + step: Step; + startedAt: number; + endedAt: number | null; + tasksCreated: number; + tasksStarted: number; + tasksFinished: number; + tasksFailed: number; + tasksNeedingRetry: number; + runsFailed: number; + runsInterrupted: number; + runsCancelled: number; + peakRunning: number; + meanRunning: number | null; + queueWaitMs: Distribution; + taskToStartMs: Distribution; + ttftMs: Distribution; + firstActivityMs: Distribution; + runDurationMs: Distribution; + board: { requests: number; failures: number; ms: Distribution }; + server: ServerMetrics; + recordedSpendUsd: number; +} + +export type CriterionStatus = "pass" | "fail" | "not_measured"; + +export interface CriterionResult { + id: string; + source: "finish-line" | "harness"; + description: string; + threshold: string; + status: CriterionStatus; + observed: number | string | null; +} + +interface Criterion { + id: string; + source: CriterionResult["source"]; + description: string; + immediate: boolean; + threshold: (t: Thresholds) => string; + evaluate: (m: StepMetrics, t: Thresholds) => { status: CriterionStatus; observed: number | string | null }; +} + +const below = (value: number | null, limit: number, strict = true) => + value === null ? { status: "not_measured" as const, observed: null } + : { status: (strict ? value < limit : value <= limit) ? "pass" as const : "fail" as const, observed: value }; + +export const CRITERIA: readonly Criterion[] = [ + { + id: "retry_needed_zero", source: "finish-line", immediate: true, + description: "Runs that need Retry (a task offered the Retry action)", + threshold: () => "= 0", + evaluate: m => ({ status: m.tasksNeedingRetry === 0 ? "pass" : "fail", observed: m.tasksNeedingRetry }), + }, + { + id: "failed_runs_zero", source: "harness", immediate: true, + description: "Runs that ended failed or timed out", + threshold: () => "= 0", + evaluate: m => ({ status: m.runsFailed === 0 ? "pass" : "fail", observed: m.runsFailed }), + }, + { + id: "tasks_rested_not_done_zero", source: "harness", immediate: false, + description: "Tasks that came to rest without being done (blocked, in review or cancelled)", + threshold: () => "= 0", + evaluate: m => ({ status: m.tasksFailed === 0 ? "pass" : "fail", observed: m.tasksFailed }), + }, + { + id: "concurrency_reached", source: "harness", immediate: false, + description: "Peak running agents against the step target", + threshold: t => `>= ${Math.round(t.minConcurrencyRatio * 100)}% of the step`, + evaluate: (m, t) => ({ status: m.peakRunning >= Math.ceil(m.step.concurrency * t.minConcurrencyRatio) ? "pass" : "fail", observed: m.peakRunning }), + }, + { + id: "board_api_p90", source: "finish-line", immediate: false, + description: "Board page API requests, p90 (ms)", + threshold: t => `< ${t.boardP90Ms} ms`, + evaluate: (m, t) => m.board.ms.n < 3 ? { status: "not_measured", observed: null } : below(m.board.ms.p90, t.boardP90Ms), + }, + { + id: "board_api_errors_zero", source: "harness", immediate: false, + description: "Board API requests that failed (no answer, 4xx or 5xx)", + threshold: () => "= 0", + evaluate: m => m.board.requests === 0 ? { status: "not_measured", observed: null } : { status: m.board.failures === 0 ? "pass" : "fail", observed: m.board.failures }, + }, + { + id: "queue_wait_p95", source: "harness", immediate: false, + description: "Queue wait before a run starts, p95 (ms)", + threshold: t => `<= ${t.queueWaitP95Ms} ms`, + evaluate: (m, t) => below(m.queueWaitMs.p95, t.queueWaitP95Ms, false), + }, + { + id: "first_activity_p95", source: "harness", immediate: false, + description: "First visible agent activity from run creation, p95 (ms), from sampled run events", + threshold: t => `<= ${t.firstActivityP95Ms} ms`, + evaluate: (m, t) => below(m.firstActivityMs.p95, t.firstActivityP95Ms, false), + }, + { + id: "main_thread_cpu", source: "finish-line", immediate: false, + description: "Main-thread CPU (or event-loop utilization) of every server process, max", + threshold: t => `< ${t.maxEventLoopUtilization}`, + evaluate: (m, t) => below(m.server.mainThreadCpuMax ?? m.server.eventLoopUtilizationMax, t.maxEventLoopUtilization), + }, + { + id: "run_event_lateness_p99", source: "finish-line", immediate: false, + description: "Run event lateness (runner emits to row committed), p99 (ms)", + threshold: t => `< ${t.runEventLatenessP99Ms} ms`, + evaluate: (m, t) => below(m.server.runEventLatenessP99MsMax, t.runEventLatenessP99Ms), + }, + { + id: "agent_api_pre_handler_p99", source: "finish-line", immediate: false, + description: "Agent API request time before the handler, p99 (ms)", + threshold: t => `< ${t.agentApiPreHandlerP99Ms} ms`, + evaluate: (m, t) => below(m.server.agentApiPreHandlerP99MsMax, t.agentApiPreHandlerP99Ms), + }, + { + id: "lease_renewals_missed_zero", source: "finish-line", immediate: false, + description: "Lease renewals that missed their deadline", + threshold: () => "= 0", + evaluate: m => m.server.leaseRenewalsMissed === null ? { status: "not_measured", observed: null } + : { status: m.server.leaseRenewalsMissed === 0 ? "pass" : "fail", observed: m.server.leaseRenewalsMissed }, + }, + { + id: "kill_one_process_resumes", source: "finish-line", immediate: false, + description: "Killing one server process: its runs resume on another, no Retry", + threshold: () => "resume, no Retry", + evaluate: () => ({ status: "not_measured", observed: "this harness does not kill server processes" }), + }, +]; + +export function evaluate(metrics: StepMetrics, thresholds: Thresholds, onlyImmediate = false): CriterionResult[] { + return CRITERIA.filter(criterion => !onlyImmediate || criterion.immediate).map(criterion => ({ + id: criterion.id, + source: criterion.source, + description: criterion.description, + threshold: criterion.threshold(thresholds), + ...criterion.evaluate(metrics, thresholds), + })); +} + +export function criteriaTable(thresholds: Thresholds): Array> { + return CRITERIA.map(criterion => ({ id: criterion.id, source: criterion.source, description: criterion.description, threshold: criterion.threshold(thresholds) })); +} + +export const firstFailure = (results: readonly CriterionResult[]) => results.find(result => result.status === "fail") ?? null; diff --git a/scripts/scale-ramp/estimate.ts b/scripts/scale-ramp/estimate.ts new file mode 100644 index 00000000000..c1513e98255 --- /dev/null +++ b/scripts/scale-ramp/estimate.ts @@ -0,0 +1,56 @@ +import type { Options } from "./config.ts"; + +export interface SpendEstimate { + agents: number; + tasks: number; + maxTasks: number; + sandboxHours: number; + modelUsd: number; + sandboxUsd: number; + totalUsd: number; + wallMinutes: number; + assumptions: string[]; +} + +const round = (value: number, places = 2) => Math.round(value * 10 ** places) / 10 ** places; + +export function estimateSpend(options: Pick): SpendEstimate { + const agents = Math.max(...options.steps.map(step => step.concurrency)); + const holdMinutes = (step: { holdMs: number }) => step.holdMs / 60_000; + const tasks = options.steps.reduce((sum, step) => sum + step.concurrency * Math.ceil(holdMinutes(step) / options.taskMinutes), 0); + const maxTasks = options.maxTasks ?? Math.ceil(tasks * 1.25); + const drainMinutes = options.drainMs / 60_000; + const sandboxHours = options.steps.reduce((sum, step) => sum + step.concurrency * holdMinutes(step) / 60, 0) + + agents * Math.min(drainMinutes, options.taskMinutes * 2) / 60; + const modelUsd = maxTasks * options.pricing.modelUsdPerTask; + const sandboxUsd = sandboxHours * options.pricing.sandboxUsdPerHour; + return { + agents, + tasks, + maxTasks, + sandboxHours: round(sandboxHours, 1), + modelUsd: round(modelUsd), + sandboxUsd: round(sandboxUsd), + totalUsd: round(modelUsd + sandboxUsd), + wallMinutes: Math.round(options.steps.reduce((sum, step) => sum + holdMinutes(step), 0) + drainMinutes), + assumptions: [ + `each agent finishes one task every ${options.taskMinutes} min, so a step creates concurrency x ceil(hold / task minutes) tasks`, + `model spend is the task cap (${maxTasks}) x $${options.pricing.modelUsdPerTask} per task (PAPERCLIP_SCALE_MODEL_USD_PER_TASK)`, + `sandbox spend is agent-hours of holds plus the drain x $${options.pricing.sandboxUsdPerHour} per sandbox-hour (PAPERCLIP_SCALE_SANDBOX_USD_PER_HOUR)`, + "these are estimates; the result reports the model spend the runs recorded", + ], + }; +} + +export type SpendDecision = { ok: true } | { ok: false; reason: string }; + +export function checkSpend(estimate: SpendEstimate, thresholdUsd: number, confirmedUsd: number | null): SpendDecision { + if (estimate.totalUsd <= thresholdUsd) return { ok: true }; + if (confirmedUsd === null) { + return { ok: false, reason: `Estimated spend $${estimate.totalUsd} is above $${thresholdUsd}; rerun with --confirm-spend of at least ${Math.ceil(estimate.totalUsd)}` }; + } + if (confirmedUsd < estimate.totalUsd) { + return { ok: false, reason: `--confirm-spend ${confirmedUsd} is below the estimated $${estimate.totalUsd}` }; + } + return { ok: true }; +} diff --git a/scripts/scale-ramp/fake-paperclip.ts b/scripts/scale-ramp/fake-paperclip.ts new file mode 100644 index 00000000000..efd17e9d02a --- /dev/null +++ b/scripts/scale-ramp/fake-paperclip.ts @@ -0,0 +1,308 @@ +import type { Clock, Fetcher } from "./client.ts"; + +type Json = Record; + +export class VirtualClock implements Clock { + t = Date.parse("2026-11-14T10:00:00.000Z"); + + now(): number { + return this.t; + } + + async sleep(ms: number): Promise { + this.t += ms; + await Promise.resolve(); + } +} + +export interface FakeConfig { + token: string; + origin: string; + cloudManaged: boolean; + startDelayMs: number; + runDurationMs: number; + capacity: number; + failWhenRunningAbove: number; + retryOnFailure: boolean; + costPerRunUsd: number; + boardLatencyMs: (running: number) => number; + failRequest: (method: string, path: string, count: number) => { status: number; body: string } | null; + metricsUrl: string | null; + metrics: (running: number) => Record | null; +} + +interface FakeRun { + id: string; + agentId: string; + issueId: string; + status: string; + createdAt: number; + startedAt: number | null; + finishedAt: number | null; + willFail: boolean; +} + +export interface RequestLog { + method: string; + url: string; + authorization: string | null; + body: string | null; +} + +const json = (status: number, body: unknown) => new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }); + +export class FakePaperclip { + readonly config: FakeConfig; + readonly requests: RequestLog[] = []; + readonly companies = new Map(); + readonly agents = new Map(); + readonly issues = new Map(); + readonly runs = new Map(); + readonly secrets = new Map(); + readonly projects = new Map(); + readonly memberCompanyIds: string[] = []; + readonly environmentId = "00000000-0000-4000-8000-0000000000e1"; + private seq = 0; + private counts = new Map(); + + readonly clock: VirtualClock; + + constructor(clock: VirtualClock, config: Partial = {}) { + this.clock = clock; + this.config = { + token: "board-token-value", + origin: "https://scale1.test.paperclip.app", + cloudManaged: false, + startDelayMs: 2_000, + runDurationMs: 60_000, + capacity: Number.POSITIVE_INFINITY, + failWhenRunningAbove: Number.POSITIVE_INFINITY, + retryOnFailure: false, + costPerRunUsd: 0.01, + boardLatencyMs: () => 40, + failRequest: () => null, + metricsUrl: null, + metrics: () => null, + ...config, + }; + } + + private id(): string { + this.seq += 1; + return `00000000-0000-4000-8000-${this.seq.toString(16).padStart(12, "0")}`; + } + + addMemberCompany(name: string): string { + const id = this.id(); + this.companies.set(id, { id, name, status: "active" }); + this.memberCompanyIds.push(id); + return id; + } + + addNoiseRuns(count: number): void { + for (let index = 0; index < count; index += 1) { + const id = this.id(); + this.runs.set(id, { id, agentId: "noise-agent", issueId: "noise-issue", status: "succeeded", createdAt: this.clock.now() + 1e10 + index, + startedAt: null, finishedAt: null, willFail: false }); + } + } + + running(): number { + return [...this.runs.values()].filter(run => run.status === "running").length; + } + + liveRuns(): FakeRun[] { + return [...this.runs.values()].filter(run => ["queued", "running"].includes(run.status)); + } + + advance(): void { + const now = this.clock.now(); + for (const run of [...this.runs.values()].sort((a, b) => a.createdAt - b.createdAt)) { + if (run.status === "running" && run.startedAt !== null && now >= run.startedAt + this.config.runDurationMs) { + run.finishedAt = run.startedAt + this.config.runDurationMs; + const issue = this.issues.get(run.issueId)!; + if (run.willFail) { + run.status = "failed"; + issue.interruption = this.config.retryOnFailure ? { state: "failed", canRetry: true, retryWait: false } : null; + } else { + run.status = "succeeded"; + issue.status = "done"; + } + } + } + for (const run of [...this.runs.values()].sort((a, b) => a.createdAt - b.createdAt)) { + if (run.status !== "queued" || now < run.createdAt + this.config.startDelayMs) continue; + if (this.running() >= this.config.capacity) break; + run.status = "running"; + run.startedAt = Math.max(run.createdAt + this.config.startDelayMs, now - 1_000); + run.willFail = this.running() > this.config.failWhenRunningAbove; + this.issues.get(run.issueId)!.status = "in_progress"; + } + } + + private runRow(run: FakeRun): Json { + const iso = (value: number | null) => value === null ? null : new Date(value).toISOString(); + return { + id: run.id, agentId: run.agentId, issueId: run.issueId, status: run.status, createdAt: iso(run.createdAt), startedAt: iso(run.startedAt), + finishedAt: iso(run.finishedAt), costUsd: run.status === "succeeded" || run.status === "failed" ? this.config.costPerRunUsd : null, + }; + } + + readonly fetch: Fetcher = async (url, init) => { + const method = String(init.method ?? "GET"); + const headers = (init.headers ?? {}) as Record; + const body = typeof init.body === "string" ? init.body : null; + this.requests.push({ method, url, authorization: headers.authorization ?? null, body }); + if (this.config.metricsUrl && url === this.config.metricsUrl) { + this.advance(); + const sample = this.config.metrics(this.running()); + return sample ? json(200, sample) : json(503, { error: "unavailable" }); + } + const parsed = new URL(url); + if (parsed.origin !== this.config.origin) throw new Error(`fake: request outside ${this.config.origin}: ${parsed.origin}`); + if (headers.authorization !== `Bearer ${this.config.token}`) return json(401, { error: "unauthorized" }); + const route = parsed.pathname; + const key = `${method} ${route.replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g, ":id")}`; + const count = (this.counts.get(key) ?? 0) + 1; + this.counts.set(key, count); + const injected = this.config.failRequest(method, key.slice(method.length + 1), count); + if (injected) return new Response(injected.body, { status: injected.status }); + this.advance(); + return this.route(method, route, parsed.searchParams, body ? JSON.parse(body) as Json : {}); + }; + + private route(method: string, route: string, query: URLSearchParams, body: Json): Response { + const parts = route.split("/").filter(Boolean).slice(1); + const now = this.clock.now(); + const [a, b, c] = parts; + if (method === "GET" && a === "cli-auth" && b === "me") return json(200, { companyIds: this.memberCompanyIds }); + if (a === "companies" && !b) { + if (method === "GET") return json(200, [...this.companies.values()]); + if (this.config.cloudManaged) return json(403, { error: "Company creation is managed by Paperclip Cloud", code: "cloud_managed" }); + const id = this.id(); + this.companies.set(id, { id, name: body.name, status: "active" }); + this.memberCompanyIds.push(id); + return json(201, this.companies.get(id)); + } + if (a === "companies" && b) { + const companyId = b; + if (!this.companies.has(companyId)) return json(404, { error: "Company not found" }); + if (!c && method === "GET") return json(200, this.companies.get(companyId)); + if (c === "archive" && method === "POST") { + this.companies.get(companyId)!.status = "archived"; + return json(200, this.companies.get(companyId)); + } + if (c === "environments") return json(200, [{ id: this.environmentId, driver: "sandbox", status: "active" }]); + if (c === "secrets") { + if (method === "GET") return json(200, [...this.secrets.values()].filter(secret => secret.companyId === companyId).map(({ value: _value, ...rest }) => rest)); + const id = this.id(); + this.secrets.set(id, { id, companyId, name: body.name, value: body.value }); + return json(201, { id, name: body.name }); + } + if (c === "projects") { + if (method === "GET") return json(200, [...this.projects.values()].filter(project => project.companyId === companyId)); + const id = this.id(); + this.projects.set(id, { id, companyId, name: body.name, archivedAt: null }); + return json(201, this.projects.get(id)); + } + if (c === "agents") { + if (method === "GET") return json(200, [...this.agents.values()].filter(agent => agent.companyId === companyId)); + const id = this.id(); + this.agents.set(id, { id, companyId, name: body.name, status: "idle", lifecycleState: null, payload: body }); + return json(201, this.agents.get(id)); + } + if (c === "issues") { + if (method === "POST") { + const id = this.id(); + this.issues.set(id, { id, companyId, title: body.title, description: body.description, status: "todo", assigneeAgentId: body.assigneeAgentId, + projectId: body.projectId ?? null, hiddenAt: null, interruption: null }); + const agent = this.agents.get(body.assigneeAgentId); + if (agent && agent.status !== "paused" && agent.status !== "terminated") { + const runId = this.id(); + this.runs.set(runId, { id: runId, agentId: body.assigneeAgentId, issueId: id, status: "queued", createdAt: now, startedAt: null, finishedAt: null, willFail: false }); + } + return json(201, this.issues.get(id)); + } + let list = [...this.issues.values()].filter(issue => issue.companyId === companyId); + const ids = query.get("ids"); + if (ids) list = list.filter(issue => ids.split(",").includes(issue.id)); + const assignee = query.get("assigneeAgentId"); + if (assignee) list = list.filter(issue => issue.assigneeAgentId === assignee); + return json(200, list.slice(0, Number(query.get("limit") ?? 500))); + } + if (c === "heartbeat-runs") { + let list = [...this.runs.values()].sort((x, y) => y.createdAt - x.createdAt); + const agentId = query.get("agentId"); + if (agentId) list = list.filter(run => run.agentId === agentId); + return json(200, list.slice(0, Math.min(1_000, Number(query.get("limit") ?? 200))).map(run => this.runRow(run))); + } + if (["dashboard", "live-runs", "sidebar-badges"].includes(c ?? "")) { + this.clock.t += this.config.boardLatencyMs(this.running()); + return json(200, c === "live-runs" ? this.liveRuns().map(run => this.runRow(run)) : {}); + } + } + if (a === "agents" && b) { + const agent = this.agents.get(b); + if (!agent) return json(404, { error: "Agent not found" }); + if (method === "PATCH") { + Object.assign(agent, body); + return json(200, agent); + } + if (method === "POST" && c === "terminate") { + agent.status = "terminated"; + agent.lifecycleState = "terminated"; + return json(200, agent); + } + if (method === "DELETE") { + this.agents.delete(b); + return json(200, { ok: true }); + } + } + if (a === "issues" && b) { + const issue = this.issues.get(b); + if (!issue) return json(404, { error: "Issue not found" }); + if (method === "GET" && !c) return json(200, issue); + if (method === "GET" && c === "runs") return json(200, [...this.runs.values()].filter(run => run.issueId === b).map(run => ({ ...this.runRow(run), runId: run.id }))); + if (method === "PATCH") { + Object.assign(issue, body); + return json(200, issue); + } + } + if (a === "heartbeat-runs" && b) { + const run = this.runs.get(b); + if (!run) return json(404, { error: "Run not found" }); + if (method === "POST" && c === "cancel") { + if (["queued", "running"].includes(run.status)) { + run.status = "cancelled"; + run.finishedAt = now; + } + return json(200, this.runRow(run)); + } + if (method === "GET" && c === "events") { + if (Number(query.get("afterSeq") ?? 0) > 0 || run.startedAt === null) return json(200, []); + const delay = run.startedAt - run.createdAt; + return json(200, [ + { seq: 1, eventType: "run.performance.span", createdAt: new Date(run.startedAt).toISOString(), payload: { span: "heartbeat.queue", startOffsetMs: 0, durationMs: delay } }, + { seq: 2, eventType: "run.performance.span", createdAt: new Date(run.startedAt + 3_000).toISOString(), + payload: { span: "task.time_to_first_agent_activity", startOffsetMs: 0, durationMs: delay + 3_000 } }, + { seq: 3, eventType: "run.performance.span", createdAt: new Date(run.startedAt + 5_000).toISOString(), + payload: { span: "task.time_to_first_token", startOffsetMs: 0, durationMs: delay + 5_000 } }, + ]); + } + } + if (a === "secrets" && b && method === "DELETE") { + this.secrets.delete(b); + return json(200, { ok: true }); + } + if (a === "projects" && b && method === "PATCH") { + const project = this.projects.get(b); + if (!project) return json(404, { error: "Project not found" }); + Object.assign(project, body); + return json(200, project); + } + if (a === "environments" && b === this.environmentId && c === "leases") { + return json(200, this.liveRuns().map(run => ({ id: run.id, issueId: run.issueId, status: "active" }))); + } + return json(404, { error: `fake: no route for ${method} ${route}` }); + } +} diff --git a/scripts/scale-ramp/guard.ts b/scripts/scale-ramp/guard.ts new file mode 100644 index 00000000000..64c4ac8b6ef --- /dev/null +++ b/scripts/scale-ramp/guard.ts @@ -0,0 +1,70 @@ +import { REFUSED_HOSTS, TEST_HOST_PATTERN } from "../../tests/restart-chaos/hosted-guard.ts"; + +export const TARGET_KINDS = ["local", "test-stack", "shared-staging", "production"] as const; +export type TargetKind = (typeof TARGET_KINDS)[number]; + +export interface Target { + baseUrl: string; + host: string; + kind: TargetKind; +} + +export class TargetRefused extends Error { + constructor(message: string) { + super(message); + this.name = "TargetRefused"; + } +} + +const LOCAL_HOSTS = new Set(["localhost", "127.0.0.1", "[::1]", "::1"]); +const RESERVED_LABELS = new Set(["admin", "admin-staging", "my", "my-staging", "api", "cloud", "www", "app"]); + +export function classifyHost(host: string): TargetKind { + const lower = host.toLowerCase(); + if (LOCAL_HOSTS.has(lower) || lower.endsWith(".localhost")) return "local"; + if (REFUSED_HOSTS.has(lower)) return "shared-staging"; + if (TEST_HOST_PATTERN.test(lower) && !RESERVED_LABELS.has(lower.split(".")[0]!)) return "test-stack"; + return "production"; +} + +export interface TargetInput { + baseUrl: string | undefined; + kind: string | undefined; + allowSharedStaging: boolean; + allowProductionHost: string | undefined; +} + +export function resolveTarget(input: TargetInput): Target { + if (!input.baseUrl) throw new TargetRefused("Set the target with --base-url or PAPERCLIP_SCALE_BASE_URL"); + if (!input.kind) throw new TargetRefused(`--target-kind is required: one of ${TARGET_KINDS.join(", ")}`); + if (!(TARGET_KINDS as readonly string[]).includes(input.kind)) { + throw new TargetRefused(`--target-kind ${input.kind} is not one of ${TARGET_KINDS.join(", ")}`); + } + let url: URL; + try { + url = new URL(input.baseUrl); + } catch { + throw new TargetRefused("The base URL is not a valid URL"); + } + if (url.username || url.password || url.search || url.hash || (url.pathname !== "/" && url.pathname !== "")) { + throw new TargetRefused("The base URL must be a bare origin, with no path, query or credentials"); + } + const host = url.hostname.toLowerCase(); + const kind = classifyHost(host); + if (kind !== input.kind) { + throw new TargetRefused(`${host} looks like a ${kind} host, not ${input.kind}; refusing`); + } + if (kind !== "local" && url.protocol !== "https:") throw new TargetRefused("A hosted target must use https"); + if (kind !== "local" && url.port) throw new TargetRefused("A hosted target must not name a port"); + if (kind === "shared-staging" && !input.allowSharedStaging) { + throw new TargetRefused(`${host} is the shared staging instance; it needs the owner's go-ahead and --allow-shared-staging`); + } + if (kind === "production" && input.allowProductionHost?.toLowerCase() !== host) { + throw new TargetRefused(`${host} looks like a production host; refusing unless --allow-production-host ${host} is also given`); + } + return { baseUrl: url.origin, host, kind }; +} + +export function assertSameOrigin(target: Target, url: string): void { + if (new URL(url).origin !== target.baseUrl) throw new TargetRefused(`Refusing a request outside ${target.baseUrl}`); +} diff --git a/scripts/scale-ramp/main.ts b/scripts/scale-ramp/main.ts new file mode 100644 index 00000000000..4099781ce2f --- /dev/null +++ b/scripts/scale-ramp/main.ts @@ -0,0 +1,180 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { createInterface } from "node:readline/promises"; +import { cleanupRun, discoverState, type CleanupReport } from "./cleanup.ts"; +import { PaperclipClient, Redactor, realClock, type Clock, type Fetcher } from "./client.ts"; +import { FALLBACK_TOKEN_ENV, parseOptions, readSecrets, TOKEN_ENV, type Options } from "./config.ts"; +import { criteriaTable } from "./criteria.ts"; +import { checkSpend, estimateSpend, type SpendEstimate } from "./estimate.ts"; +import { Ramp, type RampOutcome } from "./ramp.ts"; +import { buildResult, summaryLines, verdictOf } from "./report.ts"; +import { prepareCompany } from "./setup.ts"; +import { emptyState, fileStateStore, type RunState } from "./state.ts"; + +export interface MainDeps { + env: NodeJS.ProcessEnv; + fetcher: Fetcher; + clock: Clock; + out: (line: string) => void; + err: (line: string) => void; + confirm: (question: string) => Promise; + onInterrupt: (handler: () => void) => () => void; + random?: () => number; +} + +export const EXIT = { pass: 0, fail: 1, usage: 2, incomplete: 3, error: 4 } as const; + +export function planLines(options: Options, estimate: SpendEstimate): string[] { + const steps = options.steps.map(step => `${step.concurrency}`).join(" -> "); + const credential = options.profile.credentialKey === null ? "none" + : options.profile.providerSecretId ? `existing secret for ${options.profile.credentialKey}` + : `${options.profile.credentialKey} from the environment, stored as a company secret and deleted at cleanup`; + return [ + `Target: ${options.target.baseUrl} (${options.target.kind})`, + `Run id: ${options.label}`, + `Company: ${options.companyId ?? "a new company named for the run id"}`, + `Ramp: ${steps} agents, ${options.steps[0]!.holdMs / 60_000} min per step, drain ${options.drainMs / 60_000} min, stop on the first failed criterion`, + `Agents: up to ${estimate.agents}, ${options.profile.provider} ${options.profile.model}; credential: ${credential}`, + `Tasks: about ${estimate.tasks}, hard cap ${estimate.maxTasks}`, + `Estimated spend: $${estimate.totalUsd} (model $${estimate.modelUsd}, sandboxes $${estimate.sandboxUsd} for ${estimate.sandboxHours} sandbox-hours), about ${estimate.wallMinutes} min`, + ...estimate.assumptions.map(line => ` assumption: ${line}`), + "Criteria:", + ...criteriaTable(options.thresholds).map(criterion => ` ${criterion.id} [${criterion.source}] ${criterion.description}: ${criterion.threshold}`), + ]; +} + +async function writeOutputs(dir: string, name: string, body: unknown, summary: string[] | null, redactor: Redactor): Promise { + await mkdir(dir, { recursive: true, mode: 0o700 }); + await writeFile(path.join(dir, name), `${redactor.text(JSON.stringify(body, null, 2))}\n`, { mode: 0o600 }); + if (summary) await writeFile(path.join(dir, "SUMMARY.md"), `${redactor.text(summary.join("\n"))}\n`, { mode: 0o600 }); +} + +export async function main(argv: readonly string[], deps: MainDeps): Promise { + let options: Options; + try { + options = parseOptions(argv, deps.env, deps.clock.now(), deps.random); + } catch (error) { + deps.err(`scale ramp: ${error instanceof Error ? error.message : String(error)}`); + return EXIT.usage; + } + const secrets = readSecrets(deps.env, options.profile.providerSecretId ? null : options.profile.credentialKey); + const redactor = new Redactor([secrets.token, secrets.credential, secrets.metricsToken]); + const out = (line: string) => deps.out(redactor.text(line)); + const err = (line: string) => deps.err(redactor.text(line)); + const estimate = estimateSpend(options); + + if (options.mode !== "cleanup") for (const line of planLines(options, estimate)) out(line); + if (options.mode === "dry-run") { + out("Dry run: nothing was created or contacted."); + return EXIT.pass; + } + if (options.mode === "run") { + const spend = checkSpend(estimate, options.spendThresholdUsd, options.confirmSpendUsd); + if (!spend.ok) { + err(spend.reason); + return EXIT.usage; + } + } + if (!secrets.token) { + err(`Set ${TOKEN_ENV} (or ${FALLBACK_TOKEN_ENV}) to a board token for ${options.target.host}`); + return EXIT.usage; + } + if (!options.yes) { + const action = options.mode === "cleanup" ? `clean up ${options.label} on` : "start creating agents and tasks on"; + const answer = await deps.confirm(`Type ${options.target.host} to ${action} it: `); + if (answer.trim().toLowerCase() !== options.target.host) { + err("Confirmation did not match the target host; nothing was done."); + return EXIT.usage; + } + } + + const api = new PaperclipClient(options.target, secrets.token, deps.fetcher, deps.clock, redactor); + const store = fileStateStore(options.outDir); + + if (options.mode === "cleanup") { + let report: CleanupReport | null = null; + let error: string | null = null; + try { + const state = await store.load() ?? await discoverState(api, options.label, options.companyId, options.target.baseUrl, deps.clock.now()); + report = await cleanupRun({ api, clock: deps.clock, label: options.label, state, log: out }); + } catch (caught) { + error = caught instanceof Error ? caught.message : String(caught); + } + await writeOutputs(options.outDir, `cleanup-${deps.clock.now()}.json`, { runId: options.label, report, error }, null, redactor); + if (error) err(`Cleanup failed: ${error}`); + if (report) out(`Cleanup ${options.label}: ${report.runsCancelled} runs cancelled, ${report.tasksArchived} tasks archived, ${report.agentsDeleted} agents deleted, ${report.errors.length} errors`); + return error || (report?.errors.length ?? 0) > 0 ? EXIT.fail : EXIT.pass; + } + + let interrupted = false; + const release = deps.onInterrupt(() => { + interrupted = true; + err("Interrupt received: stopping the ramp and cleaning up"); + }); + const startedAt = deps.clock.now(); + const state: RunState = emptyState(options.label, options.target.baseUrl, startedAt); + let outcome: RampOutcome | null = null; + let cleanup: CleanupReport | null = null; + let error: string | null = null; + try { + await store.save(state); + await prepareCompany({ api, options, state, store, credential: secrets.credential, log: out }); + const ramp = new Ramp({ + api, clock: deps.clock, fetcher: deps.fetcher, options, state, store, maxTasks: estimate.maxTasks, + spendCapUsd: options.confirmSpendUsd ?? Math.max(options.spendThresholdUsd, estimate.totalUsd), metricsToken: secrets.metricsToken, + log: out, shouldStop: () => interrupted, + }); + outcome = await ramp.run(); + } catch (caught) { + error = caught instanceof Error ? caught.message : String(caught); + err(`Ramp stopped by an error: ${error}`); + } finally { + out(`Cleaning up ${options.label}`); + try { + cleanup = await cleanupRun({ api, clock: deps.clock, label: options.label, state, log: out }); + } catch (caught) { + err(`Cleanup failed; finish it with --cleanup-only ${options.label}: ${caught instanceof Error ? caught.message : String(caught)}`); + } + release(); + } + const failedRequests = api.records.filter(record => !record.ok).length; + const result = buildResult({ + options, estimate, startedAt, finishedAt: deps.clock.now(), outcome, cleanup, error, + requests: { total: api.records.length, failed: failedRequests }, + credentialSource: options.profile.credentialKey === null ? "none" : options.profile.providerSecretId ? "existing secret" : "stored for the run and deleted at cleanup", + }); + const summary = summaryLines(result); + await writeOutputs(options.outDir, "result.json", result, summary, redactor); + for (const line of summary) out(line); + out(`Result: ${path.join(options.outDir, "result.json")}`); + if (cleanup && cleanup.errors.length) err(`Cleanup left ${cleanup.errors.length} errors; finish it with --cleanup-only ${options.label}`); + const verdict = verdictOf(outcome, error); + return verdict === "pass" ? EXIT.pass : verdict === "fail" ? EXIT.fail : verdict === "incomplete" ? EXIT.incomplete : EXIT.error; +} + +export function processDeps(): MainDeps { + return { + env: process.env, + fetcher: (url, init) => fetch(url, init), + clock: realClock, + out: line => process.stdout.write(`${line}\n`), + err: line => process.stderr.write(`${line}\n`), + confirm: async question => { + const rl = createInterface({ input: process.stdin, output: process.stderr }); + try { + return await rl.question(question); + } finally { + rl.close(); + } + }, + onInterrupt: handler => { + const listener = () => handler(); + process.once("SIGINT", listener); + process.once("SIGTERM", listener); + return () => { + process.off("SIGINT", listener); + process.off("SIGTERM", listener); + }; + }, + }; +} diff --git a/scripts/scale-ramp/ramp.ts b/scripts/scale-ramp/ramp.ts new file mode 100644 index 00000000000..2a329474559 --- /dev/null +++ b/scripts/scale-ramp/ramp.ts @@ -0,0 +1,474 @@ +import { readMetrics, type Clock, type Fetcher, type PaperclipClient } from "./client.ts"; +import type { Options, Step } from "./config.ts"; +import { evaluate, firstFailure, type CriterionResult, type ServerMetrics, type StepMetrics } from "./criteria.ts"; +import { LIVE_RUN_STATUSES } from "./cleanup.ts"; +import { ensureAgents } from "./setup.ts"; +import { distribution } from "./stats.ts"; +import { mapLimit, taskTag, type RunState, type StateStore } from "./state.ts"; +import { runTiming, taskBrief, taskNonce, taskTitle, type RunEvent } from "./workload.ts"; + +type Json = Record; + +export const BOARD_MIX = [ + (companyId: string) => `/api/companies/${companyId}/dashboard`, + (companyId: string) => `/api/companies/${companyId}/issues?limit=50`, + (companyId: string) => `/api/companies/${companyId}/agents`, + (companyId: string) => `/api/companies/${companyId}/live-runs`, + (companyId: string) => `/api/companies/${companyId}/sidebar-badges`, +]; + +const FAILED_RUN = new Set(["failed", "timed_out"]); +const RESTING_NOT_DONE = new Set(["cancelled", "blocked", "in_review"]); +const RUN_PAGE = 1_000; +const IDS_PER_READ = 100; +const RETRY_CHECK_AFTER_MS = 60_000; +const RETRY_RECHECK_MS = 30_000; +const EVENT_FETCHES_PER_TICK = 10; +const EVENT_PAGES = 4; +const SINGLE_READS_PER_TICK = 50; +const AGENT_RUN_PAGE = 20; +const AGENT_RUN_READ_PARALLELISM = 8; + +interface TaskRecord { + id: string; + agentId: string; + stepIndex: number; + nonce: string; + createdAt: number; + status: string; + outcome: "open" | "done" | "failed"; + neededRetry: boolean; + lastRetryCheckAt: number | null; +} + +interface RunRecord { + id: string; + taskId: string; + agentId: string; + status: string; + createdAt: number | null; + startedAt: number | null; + finishedAt: number | null; + costUsd: number; + sampled: boolean; +} + +interface StepAccumulator { + step: Step; + startedAt: number; + endedAt: number | null; + runningSamples: number[]; + boardMs: number[]; + boardRequests: number; + boardFailures: number; + serverSamples: Array>; + ttftMs: number[]; + firstActivityMs: number[]; + sampled: number; +} + +export interface StepResult { + index: number; + concurrency: number; + holdMs: number; + metrics: StepMetrics; + criteriaAtEnd: CriterionResult[]; + criteriaFinal: CriterionResult[]; +} + +export interface StopReason { + stepIndex: number; + reason: string; + criterion: string | null; +} + +export interface RampOutcome { + steps: StepResult[]; + stop: StopReason | null; + drained: boolean; + openTasksAtEnd: number; + tasksCreated: number; + taskCapReached: boolean; + recordedSpendUsd: number; + runsListTruncated: boolean; + metricsSource: "endpoint" | "not provided" | "unavailable"; +} + +export interface RampDeps { + api: PaperclipClient; + clock: Clock; + fetcher: Fetcher; + options: Options; + state: RunState; + store: StateStore; + maxTasks: number; + spendCapUsd: number; + metricsToken: string | null; + log: (line: string) => void; + shouldStop: () => boolean; +} + +const ms = (value: unknown): number | null => { + if (value === null || value === undefined || value === "") return null; + const parsed = typeof value === "number" ? value : Date.parse(String(value)); + return Number.isFinite(parsed) ? parsed : null; +}; + +const num = (value: unknown): number | null => { + const parsed = typeof value === "number" ? value : typeof value === "string" && value !== "" ? Number(value) : NaN; + return Number.isFinite(parsed) ? parsed : null; +}; + +function rows(body: unknown): Json[] { + if (Array.isArray(body)) return body as Json[]; + if (body && typeof body === "object") { + for (const key of ["issues", "rows", "items"]) { + const value = (body as Json)[key]; + if (Array.isArray(value)) return value as Json[]; + } + } + return []; +} + +export function serverMetrics(samples: ReadonlyArray>): ServerMetrics { + const max = (key: string) => { + const values = samples.map(sample => num(sample[key])).filter((value): value is number => value !== null); + return values.length ? Math.max(...values) : null; + }; + const missed = samples.map(sample => num(sample.leaseRenewalsMissed)).filter((value): value is number => value !== null); + return { + mainThreadCpuMax: max("mainThreadCpu"), + eventLoopUtilizationMax: max("eventLoopUtilization"), + eventLoopDelayP99MsMax: max("eventLoopDelayP99Ms"), + statementsPerSecondMax: max("statementsPerSecond"), + poolWaitP99MsMax: max("poolWaitP99Ms"), + runEventLatenessP99MsMax: max("runEventLatenessP99Ms"), + agentApiPreHandlerP99MsMax: max("agentApiPreHandlerP99Ms"), + leaseRenewalsMissed: missed.length ? missed[missed.length - 1]! - missed[0]! : null, + samples: samples.length, + }; +} + +export class Ramp { + private readonly tasks = new Map(); + private readonly runs = new Map(); + private readonly runsByTask = new Map>(); + private readonly openByAgent = new Map(); + private readonly accumulators: StepAccumulator[] = []; + private nextBoardAt = 0; + private nextPollAt = 0; + private taskCapReached = false; + private fallbackCursor = 0; + private runsListTruncated = false; + private metricsSource: RampOutcome["metricsSource"]; + private readonly deps: RampDeps; + + constructor(deps: RampDeps) { + this.deps = deps; + this.metricsSource = deps.options.metricsUrl ? "unavailable" : "not provided"; + } + + private get companyId(): string { + return this.deps.state.companyId!; + } + + private current(): StepAccumulator { + return this.accumulators[this.accumulators.length - 1]!; + } + + async run(): Promise { + const { options, clock, log } = this.deps; + const results: StepResult[] = []; + let stop: StopReason | null = null; + for (const step of options.steps) { + if (this.deps.shouldStop()) { + stop = { stepIndex: step.index, reason: "interrupted before the step", criterion: null }; + break; + } + log(`Step ${step.index + 1}/${options.steps.length}: ${step.concurrency} agents for ${Math.round(step.holdMs / 60_000)} min`); + await ensureAgents({ api: this.deps.api, options, state: this.deps.state, store: this.deps.store, count: step.concurrency }); + const accumulator: StepAccumulator = { step, startedAt: clock.now(), endedAt: null, runningSamples: [], boardMs: [], boardRequests: 0, + boardFailures: 0, serverSamples: [], ttftMs: [], firstActivityMs: [], sampled: 0 }; + this.accumulators.push(accumulator); + this.nextPollAt = 0; + this.nextBoardAt = 0; + const until = accumulator.startedAt + step.holdMs; + stop = await this.loop(until, step.concurrency); + accumulator.endedAt = clock.now(); + const atEnd = evaluate(this.metrics(step.index), options.thresholds); + results.push({ index: step.index, concurrency: step.concurrency, holdMs: step.holdMs, metrics: this.metrics(step.index), criteriaAtEnd: atEnd, criteriaFinal: atEnd }); + if (stop) break; + const failed = firstFailure(atEnd); + if (failed) { + stop = { stepIndex: step.index, reason: `${failed.id} failed at the end of the step (observed ${failed.observed}, needs ${failed.threshold})`, criterion: failed.id }; + break; + } + log(`Step ${step.index + 1} passed its measured criteria`); + } + let drained = false; + if (!stop && options.drainMs > 0) { + log(`Draining open tasks for up to ${Math.round(options.drainMs / 60_000)} min`); + stop = await this.loop(clock.now() + options.drainMs, 0, true); + drained = !stop && this.openTasks().length === 0; + const last = this.current(); + last.endedAt = clock.now(); + } + for (const result of results) { + result.metrics = this.metrics(result.index); + result.criteriaFinal = evaluate(result.metrics, options.thresholds); + } + return { + steps: results, stop, drained, openTasksAtEnd: this.openTasks().length, tasksCreated: this.tasks.size, taskCapReached: this.taskCapReached, + recordedSpendUsd: this.recordedSpend(), runsListTruncated: this.runsListTruncated, metricsSource: this.metricsSource, + }; + } + + private async loop(until: number, concurrency: number, draining = false): Promise { + const { clock, options } = this.deps; + const stepIndex = this.current().step.index; + while (clock.now() < until) { + if (this.deps.shouldStop()) return { stepIndex, reason: "interrupted", criterion: null }; + if (clock.now() >= this.nextBoardAt) { + this.nextBoardAt = clock.now() + options.boardProbeMs; + await this.probeBoard(); + } + if (clock.now() >= this.nextPollAt) { + this.nextPollAt = clock.now() + options.pollMs; + await this.poll(); + const stop = this.immediateStop(stepIndex); + if (stop) return stop; + if (!draining) await this.topUp(concurrency, stepIndex, until); + else if (this.openTasks().length === 0) return null; + } + const wake = Math.min(this.nextBoardAt, this.nextPollAt, until); + await clock.sleep(Math.max(1, wake - clock.now())); + } + return null; + } + + private immediateStop(stepIndex: number): StopReason | null { + if (this.recordedSpend() > this.deps.spendCapUsd) { + return { stepIndex, reason: `recorded model spend $${this.recordedSpend().toFixed(2)} passed the cap of $${this.deps.spendCapUsd}`, criterion: "spend_cap" }; + } + for (const accumulator of this.accumulators) { + const failed = firstFailure(evaluate(this.metrics(accumulator.step.index), this.deps.options.thresholds, true)); + if (failed) return { stepIndex, reason: `${failed.id} failed (observed ${failed.observed}, needs ${failed.threshold})`, criterion: failed.id }; + } + return null; + } + + private openTasks(): TaskRecord[] { + return [...this.tasks.values()].filter(task => task.outcome === "open"); + } + + private recordedSpend(): number { + let total = 0; + for (const run of this.runs.values()) total += run.costUsd; + return Math.round(total * 100) / 100; + } + + private async topUp(concurrency: number, stepIndex: number, until: number): Promise { + const { api, state, store, options } = this.deps; + for (const agentId of state.agentIds.slice(0, concurrency)) { + if (this.deps.shouldStop() || this.deps.clock.now() >= until) return; + if (this.openByAgent.has(agentId)) continue; + if (this.tasks.size >= this.deps.maxTasks) { + this.taskCapReached = true; + return; + } + const index = state.taskIds.length; + const nonce = taskNonce(options.label, index); + const created = await api.post<{ id: string }>(`/api/companies/${this.companyId}/issues`, { + title: taskTitle(taskTag(options.label), nonce), + description: taskBrief(nonce), + status: "todo", + priority: "medium", + assigneeAgentId: agentId, + ...(state.projectId ? { projectId: state.projectId } : {}), + }); + const id = String(created.id); + state.taskIds.push(id); + await store.save(state); + this.tasks.set(id, { id, agentId, stepIndex, nonce, createdAt: this.deps.clock.now(), status: "todo", outcome: "open", neededRetry: false, lastRetryCheckAt: null }); + this.openByAgent.set(agentId, id); + } + } + + private async probeBoard(): Promise { + const accumulator = this.current(); + for (const route of BOARD_MIX) { + const before = this.deps.api.records.length; + await this.deps.api.get(route(this.companyId), "board").catch(() => null); + const record = this.deps.api.records[before]; + if (!record) continue; + accumulator.boardRequests += 1; + if (record.ok) accumulator.boardMs.push(record.ms); + else accumulator.boardFailures += 1; + } + } + + private async poll(): Promise { + await this.readRuns(); + await this.readTasks(); + await this.checkRetry(); + await this.sampleEvents(); + let running = 0; + for (const run of this.runs.values()) if (run.status === "running") running += 1; + this.current().runningSamples.push(running); + if (this.deps.options.metricsUrl) { + const sample = await readMetrics(this.deps.fetcher, this.deps.options.metricsUrl, this.deps.metricsToken, this.deps.clock); + if (sample) { + this.metricsSource = "endpoint"; + this.current().serverSamples.push(sample); + } + } + } + + private noteRun(row: Json): void { + const taskId = String(row.issueId ?? row.contextSnapshot?.issueId ?? ""); + if (!this.tasks.has(taskId)) return; + const id = String(row.id ?? row.runId); + const previous = this.runs.get(id); + this.runs.set(id, { + id, taskId, agentId: String(row.agentId), status: String(row.status), createdAt: ms(row.createdAt), startedAt: ms(row.startedAt), + finishedAt: ms(row.finishedAt), costUsd: num(row.costUsd) ?? previous?.costUsd ?? 0, sampled: previous?.sampled ?? false, + }); + const own = this.runsByTask.get(taskId) ?? new Set(); + own.add(id); + this.runsByTask.set(taskId, own); + } + + private async readRuns(): Promise { + const page = await this.deps.api.get(`/api/companies/${this.companyId}/heartbeat-runs?summary=true&limit=${RUN_PAGE}`); + for (const row of page) this.noteRun(row); + if (page.length < RUN_PAGE) return; + this.runsListTruncated = true; + const listed = new Set(page.map(row => String(row.id))); + const agents = new Set(); + for (const run of this.runs.values()) if (LIVE_RUN_STATUSES.has(run.status) && !listed.has(run.id)) agents.add(run.agentId); + for (const task of this.openTasks()) if (!(this.runsByTask.get(task.id)?.size)) agents.add(task.agentId); + await mapLimit([...agents], AGENT_RUN_READ_PARALLELISM, async agentId => { + const own = await this.deps.api.get(`/api/companies/${this.companyId}/heartbeat-runs?agentId=${agentId}&summary=true&limit=${AGENT_RUN_PAGE}`) + .catch(() => [] as Json[]); + for (const row of own) this.noteRun(row); + }); + } + + private async readTasks(): Promise { + const open = this.openTasks(); + const missing: TaskRecord[] = []; + for (let offset = 0; offset < open.length; offset += IDS_PER_READ) { + const batch = open.slice(offset, offset + IDS_PER_READ); + const body = await this.deps.api.get(`/api/companies/${this.companyId}/issues?view=rows&ids=${batch.map(task => task.id).join(",")}`).catch(() => null); + const byId = new Map(rows(body).map(row => [String(row.id), row])); + for (const task of batch) { + const row = byId.get(task.id); + if (row?.status) this.settle(task, String(row.status)); + else missing.push(task); + } + } + if (!missing.length) return; + const start = this.fallbackCursor % missing.length; + const chosen = [...missing.slice(start), ...missing.slice(0, start)].slice(0, SINGLE_READS_PER_TICK); + this.fallbackCursor += chosen.length; + for (const task of chosen) { + const row = await this.deps.api.get(`/api/issues/${task.id}`).catch(() => null); + if (row?.status) this.settle(task, String(row.status)); + } + } + + private settle(task: TaskRecord, status: string): void { + task.status = status; + if (status === "done") task.outcome = "done"; + else if (RESTING_NOT_DONE.has(status)) task.outcome = "failed"; + if (task.outcome !== "open" && this.openByAgent.get(task.agentId) === task.id) this.openByAgent.delete(task.agentId); + } + + private runsOf(taskId: string): RunRecord[] { + return [...(this.runsByTask.get(taskId) ?? [])].map(id => this.runs.get(id)!).sort((a, b) => (a.createdAt ?? 0) - (b.createdAt ?? 0)); + } + + private async checkRetry(): Promise { + const now = this.deps.clock.now(); + for (const task of this.tasks.values()) { + if (task.neededRetry || task.outcome === "done") continue; + if (task.lastRetryCheckAt !== null && now - task.lastRetryCheckAt < RETRY_RECHECK_MS) continue; + const runs = this.runsOf(task.id); + if (!runs.length || runs.some(run => LIVE_RUN_STATUSES.has(run.status))) continue; + const last = runs[runs.length - 1]!; + const idle = last.finishedAt !== null && now - last.finishedAt >= RETRY_CHECK_AFTER_MS; + if (task.outcome === "open" && last.status === "succeeded" && !idle) continue; + if (task.outcome === "failed" && task.lastRetryCheckAt !== null) continue; + task.lastRetryCheckAt = now; + const view = await this.deps.api.get(`/api/issues/${task.id}`).catch(() => null); + if (view && view.status !== "done" && view.interruption?.canRetry === true && !view.interruption?.retryWait) { + task.neededRetry = true; + this.deps.log(`Task ${task.nonce} offers Retry`); + } + } + } + + private async sampleEvents(): Promise { + const cap = this.deps.options.eventSamplePerStep; + let fetched = 0; + for (const run of this.runs.values()) { + if (fetched >= EVENT_FETCHES_PER_TICK) return; + if (run.sampled || run.status !== "succeeded") continue; + const accumulator = this.accumulators[this.tasks.get(run.taskId)!.stepIndex]!; + if (accumulator.sampled >= cap) continue; + run.sampled = true; + accumulator.sampled += 1; + fetched += 1; + const events: RunEvent[] = []; + for (let page = 0, afterSeq = 0; page < EVENT_PAGES; page += 1) { + const rowsPage = await this.deps.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=500`).catch(() => [] as Json[]); + for (const row of rowsPage) events.push({ eventType: String(row.eventType), createdAt: String(row.createdAt), payload: (row.payload ?? null) as Json | null }); + const last = Number(rowsPage.at(-1)?.seq); + if (rowsPage.length < 500 || !Number.isFinite(last)) break; + afterSeq = last; + } + const timing = runTiming(events, run.createdAt); + if (timing.ttftMs !== null) accumulator.ttftMs.push(timing.ttftMs); + if (timing.firstActivityMs !== null) accumulator.firstActivityMs.push(timing.firstActivityMs); + } + } + + metrics(stepIndex: number): StepMetrics { + const accumulator = this.accumulators[stepIndex]!; + const tasks = [...this.tasks.values()].filter(task => task.stepIndex === stepIndex); + const taskIds = new Set(tasks.map(task => task.id)); + const runs = [...this.runs.values()].filter(run => taskIds.has(run.taskId)); + const firstStart = new Map(); + for (const run of runs) { + if (run.startedAt === null) continue; + const known = firstStart.get(run.taskId); + if (known === undefined || run.startedAt < known) firstStart.set(run.taskId, run.startedAt); + } + const samples = accumulator.runningSamples; + let spend = 0; + for (const run of runs) spend += run.costUsd; + return { + step: accumulator.step, + startedAt: accumulator.startedAt, + endedAt: accumulator.endedAt, + tasksCreated: tasks.length, + tasksStarted: firstStart.size, + tasksFinished: tasks.filter(task => task.outcome === "done").length, + tasksFailed: tasks.filter(task => task.outcome === "failed").length, + tasksNeedingRetry: tasks.filter(task => task.neededRetry).length, + runsFailed: runs.filter(run => FAILED_RUN.has(run.status)).length, + runsInterrupted: runs.filter(run => run.status === "interrupted").length, + runsCancelled: runs.filter(run => run.status === "cancelled").length, + peakRunning: samples.length ? Math.max(...samples) : 0, + meanRunning: samples.length ? Math.round((samples.reduce((sum, value) => sum + value, 0) / samples.length) * 10) / 10 : null, + queueWaitMs: distribution(runs.flatMap(run => run.startedAt !== null && run.createdAt !== null ? [run.startedAt - run.createdAt] : [])), + taskToStartMs: distribution(tasks.flatMap(task => firstStart.has(task.id) ? [firstStart.get(task.id)! - task.createdAt] : [])), + ttftMs: distribution(accumulator.ttftMs), + firstActivityMs: distribution(accumulator.firstActivityMs), + runDurationMs: distribution(runs.flatMap(run => run.startedAt !== null && run.finishedAt !== null ? [run.finishedAt - run.startedAt] : [])), + board: { requests: accumulator.boardRequests, failures: accumulator.boardFailures, ms: distribution(accumulator.boardMs) }, + server: serverMetrics(accumulator.serverSamples), + recordedSpendUsd: Math.round(spend * 100) / 100, + }; + } +} diff --git a/scripts/scale-ramp/report.ts b/scripts/scale-ramp/report.ts new file mode 100644 index 00000000000..745dbc1e03a --- /dev/null +++ b/scripts/scale-ramp/report.ts @@ -0,0 +1,164 @@ +import type { CleanupReport } from "./cleanup.ts"; +import type { Options } from "./config.ts"; +import { criteriaTable, type CriterionResult } from "./criteria.ts"; +import type { SpendEstimate } from "./estimate.ts"; +import type { RampOutcome } from "./ramp.ts"; +import type { Distribution } from "./stats.ts"; + +export type Verdict = "pass" | "fail" | "incomplete" | "interrupted" | "error"; + +export interface RampResult { + schema: "paperclip.scale-ramp.result/v1"; + runId: string; + target: { baseUrl: string; host: string; kind: string }; + startedAt: string; + finishedAt: string; + plan: { + steps: Array<{ concurrency: number; holdMinutes: number }>; + drainMinutes: number; + pollSeconds: number; + boardProbeSeconds: number; + agent: { provider: string; model: string; credential: string }; + estimate: SpendEstimate; + }; + criteria: ReturnType; + verdict: Verdict; + stop: RampOutcome["stop"]; + steps: RampOutcome["steps"]; + notMeasured: string[]; + incompleteReasons: string[]; + drained: boolean; + openTasksAtEnd: number; + tasksCreated: number; + taskCapReached: boolean; + recordedSpendUsd: number | null; + sources: Record; + cleanup: CleanupReport | null; + requests: { total: number; failed: number }; + error: string | null; +} + +export function verdictOf(outcome: RampOutcome | null, error: string | null): Verdict { + if (error || !outcome) return "error"; + if (outcome.stop && outcome.stop.criterion) return "fail"; + if (outcome.stop) return "interrupted"; + if (outcome.steps.some(step => step.criteriaFinal.some(result => result.status === "fail"))) return "fail"; + if (incompleteReasons(outcome).length) return "incomplete"; + return "pass"; +} + +export function incompleteReasons(outcome: RampOutcome | null): string[] { + if (!outcome) return []; + const reasons: string[] = []; + const missing = notMeasured(outcome); + if (missing.length) reasons.push(`criteria not measured: ${missing.join(", ")}`); + if (outcome.taskCapReached) reasons.push("the task cap was reached, so a step may have run below its concurrency"); + if (outcome.openTasksAtEnd > 0) reasons.push(`${outcome.openTasksAtEnd} tasks were still open when the drain ended`); + return reasons; +} + +export function notMeasured(outcome: RampOutcome | null): string[] { + const ids = new Set(); + for (const step of outcome?.steps ?? []) for (const result of step.criteriaFinal) if (result.status === "not_measured") ids.add(result.id); + return [...ids]; +} + +export function buildResult(input: { + options: Options; + estimate: SpendEstimate; + startedAt: number; + finishedAt: number; + outcome: RampOutcome | null; + cleanup: CleanupReport | null; + error: string | null; + requests: { total: number; failed: number }; + credentialSource: string; +}): RampResult { + const { options, outcome } = input; + return { + schema: "paperclip.scale-ramp.result/v1", + runId: options.label, + target: { baseUrl: options.target.baseUrl, host: options.target.host, kind: options.target.kind }, + startedAt: new Date(input.startedAt).toISOString(), + finishedAt: new Date(input.finishedAt).toISOString(), + plan: { + steps: options.steps.map(step => ({ concurrency: step.concurrency, holdMinutes: step.holdMs / 60_000 })), + drainMinutes: options.drainMs / 60_000, + pollSeconds: options.pollMs / 1_000, + boardProbeSeconds: options.boardProbeMs / 1_000, + agent: { provider: options.profile.provider, model: options.profile.model, credential: input.credentialSource }, + estimate: input.estimate, + }, + criteria: criteriaTable(options.thresholds), + verdict: verdictOf(outcome, input.error), + stop: outcome?.stop ?? null, + steps: outcome?.steps ?? [], + notMeasured: notMeasured(outcome), + incompleteReasons: verdictOf(outcome, input.error) === "incomplete" ? incompleteReasons(outcome) : [], + drained: outcome?.drained ?? false, + openTasksAtEnd: outcome?.openTasksAtEnd ?? 0, + tasksCreated: outcome?.tasksCreated ?? 0, + taskCapReached: outcome?.taskCapReached ?? false, + recordedSpendUsd: outcome ? outcome.recordedSpendUsd : null, + sources: { + runsAndTasks: outcome?.runsListTruncated ? "API (company run list was full; open tasks without a listed run were read one by one)" : "API", + boardLatency: "harness GET mix (dashboard, issues, agents, live runs, sidebar badges)", + firstTokenAndActivity: `run events of up to ${options.eventSamplePerStep} succeeded runs per step`, + serverMetrics: outcome?.metricsSource === "endpoint" ? "PAPERCLIP_SCALE_METRICS_URL" + : outcome?.metricsSource === "unavailable" ? "PAPERCLIP_SCALE_METRICS_URL gave no readable sample: not measured" : "not provided: not measured", + database: "not read by this harness: not measured", + }, + cleanup: input.cleanup, + requests: input.requests, + error: input.error, + }; +} + +const seconds = (value: number | null) => value === null ? "not measured" : `${(value / 1_000).toFixed(1)} s`; +const millis = (value: number | null) => value === null ? "not measured" : `${Math.round(value)} ms`; +const dist = (d: Distribution, format: (value: number | null) => string) => d.n === 0 ? "not measured" : `${format(d.p50)} / ${format(d.p95)} / ${format(d.p99)} (n=${d.n})`; + +function criterionLine(result: CriterionResult): string { + const observed = result.observed === null ? "not measured" : String(result.observed); + return ` - ${result.status === "pass" ? "PASS" : result.status === "fail" ? "FAIL" : "not measured"} ${result.id}: ${observed} (needs ${result.threshold})`; +} + +export function summaryLines(result: RampResult): string[] { + const lines = [ + `# Scale ramp ${result.runId}`, + "", + `Verdict: **${result.verdict}**. Target ${result.target.baseUrl} (${result.target.kind}).`, + result.stop ? `Stopped at step ${result.stop.stepIndex + 1}: ${result.stop.reason}.` : "No stop.", + `Tasks created: ${result.tasksCreated}${result.taskCapReached ? " (task cap reached)" : ""}. Recorded model spend: ${result.recordedSpendUsd === null ? "not measured" : `$${result.recordedSpendUsd}`}; estimate was $${result.plan.estimate.totalUsd}.`, + "", + ]; + for (const step of result.steps) { + const m = step.metrics; + lines.push( + `## Step ${step.index + 1}: ${step.concurrency} agents, ${step.holdMs / 60_000} min`, + "", + `- Tasks created / started / finished / failed / needed Retry: ${m.tasksCreated} / ${m.tasksStarted} / ${m.tasksFinished} / ${m.tasksFailed} / ${m.tasksNeedingRetry}`, + `- Runs failed / interrupted / cancelled: ${m.runsFailed} / ${m.runsInterrupted} / ${m.runsCancelled}`, + `- Running agents peak / mean: ${m.peakRunning} / ${m.meanRunning ?? "not measured"}`, + `- Queue wait p50 / p95 / p99: ${dist(m.queueWaitMs, seconds)}`, + `- Time to first token p50 / p95 / p99: ${dist(m.ttftMs, seconds)}`, + `- First activity p50 / p95 / p99: ${dist(m.firstActivityMs, seconds)}`, + `- Run duration p50 / p95 / p99: ${dist(m.runDurationMs, seconds)}`, + `- Board API p50 / p95 / p99: ${dist(m.board.ms, millis)}; failed ${m.board.failures} of ${m.board.requests}`, + `- Event-loop utilization max: ${m.server.eventLoopUtilizationMax ?? "not measured"}; delay p99 max: ${millis(m.server.eventLoopDelayP99MsMax)}; statements/s max: ${m.server.statementsPerSecondMax ?? "not measured"}; pool wait p99 max: ${millis(m.server.poolWaitP99MsMax)}`, + "- Criteria (final):", + ...step.criteriaFinal.map(criterionLine), + "", + ); + } + if (result.notMeasured.length) lines.push(`Not measured: ${result.notMeasured.join(", ")}.`, ""); + if (result.incompleteReasons.length) lines.push(`Incomplete because: ${result.incompleteReasons.join("; ")}.`, ""); + lines.push(`Drain: ${result.drained ? "every task finished" : `${result.openTasksAtEnd} tasks still open`}.`, ""); + if (result.cleanup) { + const c = result.cleanup; + lines.push(`Cleanup in ${seconds(c.durationMs)}: ${c.runsCancelled} runs cancelled, ${c.tasksArchived} tasks archived, ${c.agentsDeleted} agents deleted` + + ` (${c.agentsKeptForCostHistory} kept terminated for cost history), sandbox leases still active: ${c.activeSandboxLeases ?? "not readable"}, errors: ${c.errors.length}.`); + } + if (result.error) lines.push(`Error: ${result.error}`); + return lines; +} diff --git a/scripts/scale-ramp/scale-ramp.test.ts b/scripts/scale-ramp/scale-ramp.test.ts new file mode 100644 index 00000000000..8751bde83ad --- /dev/null +++ b/scripts/scale-ramp/scale-ramp.test.ts @@ -0,0 +1,544 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { after, describe, it } from "node:test"; +import { PaperclipClient, Redactor } from "./client.ts"; +import { parseOptions, QUALIFIED_OPENCODE_MODEL } from "./config.ts"; +import { evaluate, type StepMetrics } from "./criteria.ts"; +import { checkSpend, estimateSpend } from "./estimate.ts"; +import { FakePaperclip, VirtualClock, type FakeConfig } from "./fake-paperclip.ts"; +import { classifyHost, resolveTarget, TargetRefused } from "./guard.ts"; +import { EXIT, main, type MainDeps } from "./main.ts"; +import { Ramp } from "./ramp.ts"; +import { verdictOf, type RampResult } from "./report.ts"; +import { ensureAgents, prepareCompany } from "./setup.ts"; +import { distribution } from "./stats.ts"; +import { emptyState, fileStateStore, mapLimit, type RunState } from "./state.ts"; +import { runTiming } from "./workload.ts"; + +const TOKEN = "board-token-SECRET-4f2a9c"; +const PROVIDER_KEY = "sk-or-v1-SECRET-provider-key-77"; +const METRICS_TOKEN = "metrics-token-SECRET-91"; +const HOST = "scale1.test.paperclip.app"; +const ORIGIN = `https://${HOST}`; +const METRICS_URL = "https://metrics.scale1.test/metrics"; + +const dirs: string[] = []; +after(async () => { + for (const dir of dirs) await rm(dir, { recursive: true, force: true }); +}); + +async function tempDir(): Promise { + const dir = await mkdtemp(path.join(tmpdir(), "scale-ramp-test-")); + dirs.push(dir); + return dir; +} + +interface Harness { + clock: VirtualClock; + fake: FakePaperclip; + out: string[]; + err: string[]; + deps: MainDeps; + fetchCalls: () => number; + interrupt: () => void; +} + +function harness(config: Partial = {}, env: Record = {}, answer = HOST): Harness { + const clock = new VirtualClock(); + const fake = new FakePaperclip(clock, { token: TOKEN, origin: ORIGIN, ...config }); + const out: string[] = []; + const err: string[] = []; + let calls = 0; + let handler: (() => void) | null = null; + const deps: MainDeps = { + env: { PAPERCLIP_SCALE_TOKEN: TOKEN, OPENROUTER_API_KEY: PROVIDER_KEY, ...env }, + fetcher: async (url, init) => { + calls += 1; + return fake.fetch(url, init); + }, + clock, + out: line => out.push(line), + err: line => err.push(line), + confirm: async () => answer, + onInterrupt: fn => { + handler = fn; + return () => { + handler = null; + }; + }, + random: () => 0.5, + }; + return { clock, fake, out, err, deps, fetchCalls: () => calls, interrupt: () => handler?.() }; +} + +const baseArgs = (out: string, extra: string[] = []) => [ + "--base-url", ORIGIN, "--target-kind", "test-stack", "--out", out, "--label", "scale-test-run", "--yes", + "--hold-minutes", "2", "--drain-minutes", "3", "--poll-seconds", "15", "--board-probe-seconds", "10", "--task-minutes", "1", ...extra, +]; + +async function readResult(out: string): Promise { + return JSON.parse(await readFile(path.join(out, "scale-test-run", "result.json"), "utf8")) as RampResult; +} + +function assertCleanedUp(fake: FakePaperclip): void { + assert.equal(fake.liveRuns().length, 0, "no queued or running run is left"); + assert.equal(fake.agents.size, 0, "every harness agent is deleted"); + assert.ok([...fake.issues.values()].every(issue => issue.hiddenAt), "every task is archived"); + assert.equal(fake.secrets.size, 0, "the stored provider secret is deleted"); + assert.ok([...fake.projects.values()].every(project => project.archivedAt), "the project is archived"); +} + +const goodMetrics = (running: number) => ({ + eventLoopUtilization: 0.2 + running / 1_000, mainThreadCpu: 0.25, eventLoopDelayP99Ms: 20, statementsPerSecond: 40 * running, + poolWaitP99Ms: 2, runEventLatenessP99Ms: 120, agentApiPreHandlerP99Ms: 15, leaseRenewalsMissed: 0, +}); + +describe("target guard", () => { + it("classifies hosts", () => { + assert.equal(classifyHost("localhost"), "local"); + assert.equal(classifyHost("app.localhost"), "local"); + assert.equal(classifyHost("127.0.0.1"), "local"); + assert.equal(classifyHost("scale1.test.paperclip.app"), "test-stack"); + assert.equal(classifyHost("chaos-a.staging.paperclip.app"), "test-stack"); + assert.equal(classifyHost("paperclip.staging.paperclip.app"), "shared-staging"); + assert.equal(classifyHost("admin.staging.paperclip.app"), "production"); + assert.equal(classifyHost("paperclip.app"), "production"); + assert.equal(classifyHost("acme.paperclip.app"), "production"); + assert.equal(classifyHost("example.com"), "production"); + }); + + it("requires a target kind that matches the host", () => { + const base = { allowSharedStaging: false, allowProductionHost: undefined }; + assert.throws(() => resolveTarget({ ...base, baseUrl: ORIGIN, kind: undefined }), /--target-kind is required/); + assert.throws(() => resolveTarget({ ...base, baseUrl: "https://acme.paperclip.app", kind: "test-stack" }), /looks like a production host/); + assert.throws(() => resolveTarget({ ...base, baseUrl: ORIGIN, kind: "local" }), /looks like a test-stack host/); + assert.throws(() => resolveTarget({ ...base, baseUrl: `http://${HOST}`, kind: "test-stack" }), /must use https/); + assert.throws(() => resolveTarget({ ...base, baseUrl: `${ORIGIN}/api`, kind: "test-stack" }), /bare origin/); + assert.deepEqual(resolveTarget({ ...base, baseUrl: "http://localhost:3100", kind: "local" }), { baseUrl: "http://localhost:3100", host: "localhost", kind: "local" }); + }); + + it("refuses production and shared staging without the second explicit flag", () => { + const base = { allowSharedStaging: false, allowProductionHost: undefined }; + assert.throws(() => resolveTarget({ ...base, baseUrl: "https://acme.paperclip.app", kind: "production" }), TargetRefused); + assert.throws(() => resolveTarget({ ...base, baseUrl: "https://acme.paperclip.app", kind: "production", allowProductionHost: "other.paperclip.app" }), TargetRefused); + assert.equal(resolveTarget({ ...base, baseUrl: "https://acme.paperclip.app", kind: "production", allowProductionHost: "acme.paperclip.app" }).kind, "production"); + assert.throws(() => resolveTarget({ ...base, baseUrl: "https://paperclip.staging.paperclip.app", kind: "shared-staging" }), /owner's go-ahead/); + assert.equal(resolveTarget({ ...base, allowSharedStaging: true, baseUrl: "https://paperclip.staging.paperclip.app", kind: "shared-staging" }).kind, "shared-staging"); + }); + + it("stops before any request when the host looks like production", async () => { + const out = await tempDir(); + const h = harness(); + const code = await main(["--base-url", "https://acme.paperclip.app", "--target-kind", "production", "--out", out, "--yes"], h.deps); + assert.equal(code, EXIT.usage); + assert.equal(h.fetchCalls(), 0); + assert.match(h.err.join("\n"), /looks like a production host/); + }); + + it("stops before any request when the typed confirmation does not match", async () => { + const out = await tempDir(); + const h = harness({}, {}, "yes"); + const args = baseArgs(out).filter(arg => arg !== "--yes"); + assert.equal(await main(args, h.deps), EXIT.usage); + assert.equal(h.fetchCalls(), 0); + }); +}); + +describe("dry run and spend", () => { + it("prints the plan and estimate and touches nothing", async () => { + const out = await tempDir(); + const h = harness(); + const code = await main(["--base-url", ORIGIN, "--target-kind", "test-stack", "--out", out, "--dry-run"], h.deps); + assert.equal(code, EXIT.pass); + assert.equal(h.fetchCalls(), 0); + assert.deepEqual(await readdir(out), []); + const text = h.out.join("\n"); + assert.match(text, /Ramp: 10 -> 50 -> 100 -> 200 -> 500 -> 1000 agents/); + assert.match(text, /Estimated spend: \$\d/); + assert.match(text, /retry_needed_zero \[finish-line\]/); + assert.match(text, /Dry run: nothing was created or contacted/); + }); + + it("estimates tasks, sandbox-hours and cost from the steps", () => { + const options = parseOptions(["--base-url", ORIGIN, "--target-kind", "test-stack", "--steps", "10,50", "--hold-minutes", "10", "--task-minutes", "5", + "--drain-minutes", "10"], {}, 0); + const estimate = estimateSpend(options); + assert.equal(estimate.agents, 50); + assert.equal(estimate.tasks, 10 * 2 + 50 * 2); + assert.equal(estimate.maxTasks, 150); + assert.equal(estimate.sandboxHours, 18.3); + assert.equal(estimate.modelUsd, 3); + assert.equal(estimate.totalUsd, 4.83); + }); + + it("requires --confirm-spend of at least the estimate above the threshold", async () => { + const estimate = { totalUsd: 120 } as ReturnType; + assert.equal(checkSpend(estimate, 5, null).ok, false); + assert.equal(checkSpend(estimate, 5, 100).ok, false); + assert.equal(checkSpend(estimate, 5, 130).ok, true); + assert.equal(checkSpend(estimate, 200, null).ok, true); + const out = await tempDir(); + const h = harness(); + const code = await main(["--base-url", ORIGIN, "--target-kind", "test-stack", "--out", out, "--yes"], h.deps); + assert.equal(code, EXIT.usage); + assert.equal(h.fetchCalls(), 0); + assert.match(h.err.join("\n"), /--confirm-spend/); + }); + + it("defaults to the qualified OpenCode model of the runner", async () => { + const source = await readFile(path.join(import.meta.dirname, "../../packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts"), "utf8"); + const match = /QUALIFIED_OPENCODE_MODEL\s*=\s*"([^"]+)"/.exec(source); + assert.equal(match?.[1], QUALIFIED_OPENCODE_MODEL); + }); +}); + +describe("criteria", () => { + const empty = distribution([]); + const metrics = (patch: Partial = {}): StepMetrics => ({ + step: { index: 0, concurrency: 10, holdMs: 60_000 }, startedAt: 0, endedAt: 1, tasksCreated: 10, tasksStarted: 10, tasksFinished: 10, + tasksFailed: 0, tasksNeedingRetry: 0, runsFailed: 0, runsInterrupted: 0, runsCancelled: 0, peakRunning: 10, meanRunning: 9, + queueWaitMs: distribution([1_000, 2_000]), taskToStartMs: empty, ttftMs: empty, firstActivityMs: distribution([4_000]), runDurationMs: empty, + board: { requests: 10, failures: 0, ms: distribution([100, 120, 130, 140]) }, + server: { mainThreadCpuMax: null, eventLoopUtilizationMax: null, eventLoopDelayP99MsMax: null, statementsPerSecondMax: null, poolWaitP99MsMax: null, + runEventLatenessP99MsMax: null, agentApiPreHandlerP99MsMax: null, leaseRenewalsMissed: null, samples: 0 }, + recordedSpendUsd: 0, ...patch, + }); + const thresholds = parseOptions(["--base-url", ORIGIN, "--target-kind", "test-stack"], {}, 0).thresholds; + const byId = (results: ReturnType) => Object.fromEntries(results.map(result => [result.id, result.status])); + + it("reports missing server metrics as not measured, never as zero", () => { + const results = byId(evaluate(metrics(), thresholds)); + assert.equal(results.retry_needed_zero, "pass"); + assert.equal(results.board_api_p90, "pass"); + assert.equal(results.main_thread_cpu, "not_measured"); + assert.equal(results.lease_renewals_missed_zero, "not_measured"); + assert.equal(results.run_event_lateness_p99, "not_measured"); + assert.equal(results.kill_one_process_resumes, "not_measured"); + assert.equal(byId(evaluate(metrics({ board: { requests: 0, failures: 0, ms: empty } }), thresholds)).board_api_p90, "not_measured"); + }); + + it("fails each finish-line threshold it can measure", () => { + const server = { mainThreadCpuMax: 0.7, eventLoopUtilizationMax: 0.5, eventLoopDelayP99MsMax: 10, statementsPerSecondMax: 1, poolWaitP99MsMax: 1, + runEventLatenessP99MsMax: 1_500, agentApiPreHandlerP99MsMax: 100, leaseRenewalsMissed: 2, samples: 3 }; + const results = byId(evaluate(metrics({ tasksNeedingRetry: 1, peakRunning: 8, board: { requests: 4, failures: 1, ms: distribution([100, 600, 700]) }, server }), thresholds)); + assert.deepEqual(results, { + retry_needed_zero: "fail", failed_runs_zero: "pass", tasks_rested_not_done_zero: "pass", concurrency_reached: "fail", board_api_p90: "fail", board_api_errors_zero: "fail", + queue_wait_p95: "pass", first_activity_p95: "pass", main_thread_cpu: "fail", run_event_lateness_p99: "fail", agent_api_pre_handler_p99: "fail", + lease_renewals_missed_zero: "fail", kill_one_process_resumes: "not_measured", + }); + }); + + it("only evaluates retry and failed runs as immediate stops", () => { + assert.deepEqual(evaluate(metrics(), thresholds, true).map(result => result.id), ["retry_needed_zero", "failed_runs_zero"]); + }); + + it("reads first token and first activity from run spans, with an event fallback", () => { + const created = Date.parse("2026-11-14T10:00:00Z"); + const spans = runTiming([ + { eventType: "run.performance.span", createdAt: "2026-11-14T10:00:01Z", payload: { span: "heartbeat.queue", startOffsetMs: 200, durationMs: 800 } }, + { eventType: "run.performance.span", createdAt: "2026-11-14T10:00:06Z", payload: { span: "task.time_to_first_token", startOffsetMs: 0, durationMs: 6_200 } }, + ], created); + assert.deepEqual(spans, { ttftMs: 6_000, firstActivityMs: null }); + const fallback = runTiming([{ eventType: "tool.execution.started", createdAt: "2026-11-14T10:00:03Z" }, { eventType: "item.delta", createdAt: "2026-11-14T10:00:07Z" }], created); + assert.deepEqual(fallback, { ttftMs: 7_000, firstActivityMs: 3_000 }); + }); +}); + +describe("ramp against the fake API", () => { + it("ramps every step, measures it, passes with a metrics endpoint and cleans up", async () => { + const out = await tempDir(); + const h = harness({ metricsUrl: METRICS_URL, metrics: goodMetrics }, { PAPERCLIP_SCALE_METRICS_URL: METRICS_URL, PAPERCLIP_SCALE_METRICS_TOKEN: METRICS_TOKEN }); + const code = await main(baseArgs(out, ["--steps", "2,4"]), h.deps); + const result = await readResult(out); + assert.equal(result.verdict, "incomplete", "the process-kill criterion is never measured here"); + assert.equal(code, EXIT.incomplete); + assert.deepEqual(result.notMeasured, ["kill_one_process_resumes"]); + assert.equal(result.steps.length, 2); + const [first, second] = result.steps; + assert.equal(first!.concurrency, 2); + assert.equal(second!.metrics.peakRunning, 4); + assert.ok(first!.metrics.tasksFinished >= 2); + assert.equal(second!.metrics.tasksNeedingRetry, 0); + assert.ok(second!.metrics.firstActivityMs.n > 0); + assert.ok(second!.metrics.board.ms.n >= 3); + assert.equal(second!.metrics.server.statementsPerSecondMax, 160); + assert.ok(second!.criteriaFinal.filter(criterion => criterion.id !== "kill_one_process_resumes").every(criterion => criterion.status === "pass")); + assert.equal(result.sources.serverMetrics, "PAPERCLIP_SCALE_METRICS_URL"); + assert.ok(result.recordedSpendUsd! > 0); + const agents = h.fake.requests.filter(request => request.method === "POST" && new URL(request.url).pathname.endsWith("/agents")); + assert.equal(agents.length, 4, "agents are created per step up to the largest step"); + const payload = JSON.parse(agents[0]!.body!); + assert.equal(payload.adapterConfig.model, QUALIFIED_OPENCODE_MODEL); + assert.equal(payload.runtimeConfig.heartbeat.maxConcurrentRuns, 1); + assert.equal(payload.adapterConfig.env.OPENROUTER_API_KEY.type, "secret_ref"); + assertCleanedUp(h.fake); + assert.ok([...h.fake.companies.values()].every(company => company.status === "archived"), "the created company is archived"); + assert.equal(result.cleanup!.errors.length, 0); + assert.equal(result.cleanup!.activeSandboxLeases, 0); + }); + + it("reports missing optional sources as not measured", async () => { + const out = await tempDir(); + const h = harness(); + await main(baseArgs(out, ["--steps", "2"]), h.deps); + const result = await readResult(out); + assert.equal(result.verdict, "incomplete"); + assert.match(result.sources.serverMetrics!, /not measured/); + assert.ok(result.notMeasured.includes("main_thread_cpu")); + const summary = await readFile(path.join(out, "scale-test-run", "SUMMARY.md"), "utf8"); + assert.match(summary, /Event-loop utilization max: not measured/); + assert.doesNotMatch(summary, /Event-loop utilization max: 0\b/); + }); + + it("stops on the first failed criterion and never starts the next step", async () => { + const out = await tempDir(); + const h = harness({ failWhenRunningAbove: 3 }); + const code = await main(baseArgs(out, ["--steps", "2,4,8"]), h.deps); + const result = await readResult(out); + assert.equal(code, EXIT.fail); + assert.equal(result.verdict, "fail"); + assert.equal(result.stop!.stepIndex, 1); + assert.equal(result.stop!.criterion, "failed_runs_zero"); + assert.equal(result.steps.length, 2); + const agents = h.fake.requests.filter(request => request.method === "POST" && new URL(request.url).pathname.endsWith("/agents")); + assert.equal(agents.length, 4, "the 8-agent step never created agents"); + assertCleanedUp(h.fake); + }); + + it("stops when a task offers Retry", async () => { + const out = await tempDir(); + const h = harness({ failWhenRunningAbove: 1, retryOnFailure: true }); + await main(baseArgs(out, ["--steps", "2,4"]), h.deps); + const result = await readResult(out); + assert.equal(result.verdict, "fail"); + assert.equal(result.stop!.stepIndex, 0); + assert.ok(["retry_needed_zero", "failed_runs_zero"].includes(result.stop!.criterion!)); + assert.ok(result.steps[0]!.metrics.tasksNeedingRetry + result.steps[0]!.metrics.runsFailed > 0); + }); + + it("fails a step whose board requests are slow", async () => { + const out = await tempDir(); + const h = harness({ boardLatencyMs: running => running >= 3 ? 900 : 50 }); + await main(baseArgs(out, ["--steps", "2,4"]), h.deps); + const result = await readResult(out); + assert.equal(result.verdict, "fail"); + assert.equal(result.stop!.criterion, "board_api_p90"); + assert.equal(result.stop!.stepIndex, 1); + }); + + it("fails a step that cannot reach its concurrency", async () => { + const out = await tempDir(); + const h = harness({ capacity: 2 }); + await main(baseArgs(out, ["--steps", "2,4"]), h.deps); + const result = await readResult(out); + assert.equal(result.stop!.criterion, "concurrency_reached"); + assert.equal(result.steps[1]!.metrics.peakRunning, 2); + }); + + it("stops creating tasks at the task cap and grades the run incomplete", async () => { + const out = await tempDir(); + const h = harness({ metricsUrl: METRICS_URL, metrics: goodMetrics }, { PAPERCLIP_SCALE_METRICS_URL: METRICS_URL }); + await main(baseArgs(out, ["--steps", "2", "--max-tasks", "3"]), h.deps); + const result = await readResult(out); + assert.equal(result.tasksCreated, 3); + assert.equal(result.taskCapReached, true); + assert.equal(result.verdict, "incomplete"); + assert.ok(result.incompleteReasons.some(reason => reason.includes("task cap"))); + }); + + it("grades a run incomplete when tasks are still open after the drain", async () => { + const out = await tempDir(); + const h = harness({ runDurationMs: 20 * 60_000 }); + await main(baseArgs(out, ["--steps", "2", "--min-concurrency-ratio", "0.5"]), h.deps); + const result = await readResult(out); + assert.equal(result.drained, false); + assert.ok(result.openTasksAtEnd > 0); + assert.ok(result.incompleteReasons.some(reason => reason.includes("still open"))); + assertCleanedUp(h.fake); + }); + + it("creates no more tasks once interrupted", async () => { + const out = await tempDir(); + const h = harness(); + const original = h.deps.fetcher; + h.deps.fetcher = async (url, init) => { + if (init.method === "POST" && new URL(url).pathname.endsWith("/issues")) h.interrupt(); + return original(url, init); + }; + await main(baseArgs(out, ["--steps", "4"]), h.deps); + assert.equal(h.fake.issues.size, 1); + assertCleanedUp(h.fake); + }); + + it("keeps tracking its runs when the company run list is full of other runs", async () => { + const out = await tempDir(); + const h = harness({ metricsUrl: METRICS_URL, metrics: goodMetrics }, { PAPERCLIP_SCALE_METRICS_URL: METRICS_URL }); + h.fake.addNoiseRuns(1_000); + await main(baseArgs(out, ["--steps", "2"]), h.deps); + const result = await readResult(out); + assert.match(result.sources.runsAndTasks!, /full/); + assert.equal(result.steps[0]!.metrics.peakRunning, 2); + assert.ok(result.steps[0]!.metrics.tasksFinished >= 2); + assert.ok(result.recordedSpendUsd! > 0); + assert.ok(result.drained, "finished runs are not left counted as running"); + }); + + it("cleans up after a crash in the middle of the ramp", async () => { + const out = await tempDir(); + const h = harness({ failRequest: (method, route, count) => method === "POST" && route.endsWith("/issues") && count === 5 ? { status: 500, body: "boom" } : null }); + const code = await main(baseArgs(out, ["--steps", "2,4"]), h.deps); + const result = await readResult(out); + assert.equal(code, EXIT.error); + assert.equal(result.verdict, "error"); + assert.match(result.error!, /returned 500/); + assertCleanedUp(h.fake); + }); + + it("cleans up when interrupted", async () => { + const out = await tempDir(); + const h = harness(); + const original = h.deps.fetcher; + let requests = 0; + h.deps.fetcher = async (url, init) => { + requests += 1; + if (requests === 40) h.interrupt(); + return original(url, init); + }; + await main(baseArgs(out, ["--steps", "2,4"]), h.deps); + const result = await readResult(out); + assert.equal(result.verdict, "interrupted"); + assertCleanedUp(h.fake); + }); + + it("finishes a cleanup with --cleanup-only after the harness process died", async () => { + const out = await tempDir(); + const h = harness(); + const options = parseOptions(baseArgs(out, ["--steps", "2"]), h.deps.env, h.clock.now()); + const api = new PaperclipClient(options.target, TOKEN, h.deps.fetcher, h.clock, new Redactor([TOKEN])); + const store = fileStateStore(options.outDir); + const state: RunState = emptyState(options.label, options.target.baseUrl, h.clock.now()); + await prepareCompany({ api, options, state, store, credential: PROVIDER_KEY, log: () => {} }); + let ticks = 0; + await new Ramp({ api, clock: h.clock, fetcher: h.deps.fetcher, options, state, store, maxTasks: 100, spendCapUsd: 100, metricsToken: null, + log: () => {}, shouldStop: () => (ticks += 1) > 6 }).run(); + assert.ok(h.fake.liveRuns().length > 0, "the dead harness left live runs"); + assert.ok(h.fake.agents.size > 0); + const code = await main(["--base-url", ORIGIN, "--target-kind", "test-stack", "--out", out, "--cleanup-only", "scale-test-run", "--yes"], h.deps); + assert.equal(code, EXIT.pass); + assertCleanedUp(h.fake); + assert.ok([...h.fake.companies.values()].every(company => company.status === "archived")); + }); + + it("archives a task the state file never recorded", async () => { + const out = await tempDir(); + const h = harness(); + const options = parseOptions(baseArgs(out, ["--steps", "2"]), h.deps.env, h.clock.now()); + const api = new PaperclipClient(options.target, TOKEN, h.deps.fetcher, h.clock, new Redactor([TOKEN])); + const store = fileStateStore(options.outDir); + const state: RunState = emptyState(options.label, options.target.baseUrl, h.clock.now()); + await prepareCompany({ api, options, state, store, credential: PROVIDER_KEY, log: () => {} }); + await ensureAgents({ api, options, state, store, count: 2 }); + const recorded = await api.post<{ id: string }>(`/api/companies/${state.companyId}/issues`, { title: "recorded", assigneeAgentId: state.agentIds[0] }); + state.taskIds.push(recorded.id); + await store.save(state); + const unrecorded = await api.post<{ id: string }>(`/api/companies/${state.companyId}/issues`, { title: "unrecorded", assigneeAgentId: state.agentIds[1] }); + assert.equal(await main(["--base-url", ORIGIN, "--target-kind", "test-stack", "--out", out, "--cleanup-only", "scale-test-run", "--yes"], h.deps), EXIT.pass); + assert.ok(h.fake.issues.get(unrecorded.id)!.hiddenAt); + assertCleanedUp(h.fake); + }); + + it("finds the run by name for --cleanup-only when the state file is gone", async () => { + const out = await tempDir(); + const h = harness(); + const options = parseOptions(baseArgs(out, ["--steps", "2"]), h.deps.env, h.clock.now()); + const api = new PaperclipClient(options.target, TOKEN, h.deps.fetcher, h.clock, new Redactor([TOKEN])); + const state: RunState = emptyState(options.label, options.target.baseUrl, h.clock.now()); + const otherCompany = h.fake.addMemberCompany("Someone else"); + await prepareCompany({ api, options, state, store: fileStateStore(path.join(out, "discarded")), credential: PROVIDER_KEY, log: () => {} }); + await api.post(`/api/companies/${otherCompany}/agents`, { name: "Unrelated agent" }); + const code = await main(["--base-url", ORIGIN, "--target-kind", "test-stack", "--out", out, "--cleanup-only", "scale-test-run", "--yes"], h.deps); + assert.equal(code, EXIT.pass); + assert.equal(h.fake.secrets.size, 0); + assert.deepEqual([...h.fake.agents.values()].map(agent => agent.name), ["Unrelated agent"], "only objects carrying the run id are touched"); + assert.equal(h.fake.companies.get(otherCompany)!.status, "active"); + }); + + it("uses a supplied company when the instance cannot create one", async () => { + const out = await tempDir(); + const h = harness({ cloudManaged: true }); + const refused = await main(baseArgs(out, ["--steps", "2"]), h.deps); + assert.equal(refused, EXIT.error); + assert.match((await readResult(out)).error!, /pass --company/); + const companyId = h.fake.addMemberCompany("Existing workspace"); + const out2 = await tempDir(); + const code = await main(baseArgs(out2, ["--steps", "2", "--company", companyId]), h.deps); + assert.equal(code, EXIT.incomplete); + assertCleanedUp(h.fake); + assert.equal(h.fake.companies.get(companyId)!.status, "active", "a supplied company is never archived"); + }); +}); + +describe("state and parallel work", () => { + it("stops assigning work after a failure and waits for started work", async () => { + const started: number[] = []; + let running = 0; + let finishedAfterFailure = 0; + let failed = false; + await assert.rejects(mapLimit([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 3, async item => { + started.push(item); + running += 1; + await new Promise(resolve => setTimeout(resolve, item === 2 ? 1 : 10)); + running -= 1; + if (item === 2) { + failed = true; + throw new Error("create failed"); + } + if (failed) finishedAfterFailure += 1; + }), /create failed/); + assert.equal(running, 0, "every started worker finished before the error surfaced"); + assert.deepEqual(started, [1, 2, 3], "no item was started after the failure"); + assert.equal(finishedAfterFailure, 2); + }); + + it("keeps state.json readable under overlapping saves", async () => { + const dir = await tempDir(); + const store = fileStateStore(dir); + const state = emptyState("scale-test-run", ORIGIN, 0); + await Promise.all(Array.from({ length: 25 }, (_, index) => store.save({ ...state, agentIds: Array.from({ length: index + 1 }, (_, n) => `agent-${n}`) }))); + const saved = await store.load(); + assert.equal(saved!.agentIds.length, 25); + assert.deepEqual(await readdir(dir), ["state.json"]); + }); +}); + +describe("credentials", () => { + it("never appear in output, results or state files", async () => { + const out = await tempDir(); + const h = harness({ + metricsUrl: METRICS_URL, metrics: goodMetrics, + failRequest: (method, route, count) => method === "GET" && route.endsWith("/sidebar-badges") && count === 1 + ? { status: 400, body: `bad request with Authorization: Bearer ${TOKEN} and key ${PROVIDER_KEY}` } : null, + }, { PAPERCLIP_SCALE_METRICS_URL: METRICS_URL, PAPERCLIP_SCALE_METRICS_TOKEN: METRICS_TOKEN }); + await main(baseArgs(out, ["--steps", "2"]), h.deps); + const files = await readdir(path.join(out, "scale-test-run")); + assert.ok(files.includes("result.json") && files.includes("state.json") && files.includes("SUMMARY.md")); + const written = await Promise.all(files.map(file => readFile(path.join(out, "scale-test-run", file), "utf8"))); + const everything = [...h.out, ...h.err, ...written].join("\n"); + for (const secret of [TOKEN, PROVIDER_KEY, METRICS_TOKEN]) assert.ok(!everything.includes(secret), `output leaked ${secret.slice(0, 12)}`); + assert.ok(h.fake.requests.some(request => request.body?.includes(PROVIDER_KEY)), "the key reaches the instance only through its secrets API"); + assert.ok(h.fake.requests.filter(request => request.body?.includes(PROVIDER_KEY)).every(request => new URL(request.url).pathname.endsWith("/secrets"))); + }); + + it("redacts bearer tokens and known values", () => { + const redactor = new Redactor([TOKEN, null, "abc"]); + assert.equal(redactor.text(`x ${TOKEN} y`), "x [redacted] y"); + assert.equal(redactor.text("Authorization: Bearer abcdefghijklmnop"), "Authorization: Bearer [redacted]"); + assert.equal(redactor.text("abc"), "abc"); + }); + + it("scores an empty outcome as an error", () => { + assert.equal(verdictOf(null, "boom"), "error"); + }); +}); diff --git a/scripts/scale-ramp/setup.ts b/scripts/scale-ramp/setup.ts new file mode 100644 index 00000000000..b442f2c859f --- /dev/null +++ b/scripts/scale-ramp/setup.ts @@ -0,0 +1,140 @@ +import { ApiError, type PaperclipClient } from "./client.ts"; +import type { AgentProfile, Options } from "./config.ts"; +import { agentName, companyName, mapLimit, projectName, secretPrefix, type RunState, type StateStore } from "./state.ts"; + +type Json = Record; + +export class SetupError extends Error { + constructor(message: string) { + super(message); + this.name = "SetupError"; + } +} + +export async function resolveCompany(api: PaperclipClient, options: Pick, state: RunState, store: StateStore): Promise { + if (options.companyId) { + const me = await api.get<{ companyIds?: string[] }>("/api/cli-auth/me"); + if (!(me.companyIds ?? []).includes(options.companyId)) throw new SetupError(`The signed-in board user is not a member of company ${options.companyId}`); + state.companyId = options.companyId; + await store.save(state); + return; + } + try { + const company = await api.post<{ id: string }>("/api/companies", { + name: companyName(options.label), + description: `Scale ramp ${options.label}. Created and archived by scripts/scale-agents-ramp.ts.`, + }); + state.companyId = String(company.id); + state.companyCreated = true; + await store.save(state); + } catch (error) { + if (error instanceof ApiError && error.status === 403) { + throw new SetupError(`This instance does not let the harness create a company (${error.code ?? "forbidden"}); pass --company or PAPERCLIP_SCALE_COMPANY_ID`); + } + throw error; + } +} + +export async function pickEnvironment(api: PaperclipClient, companyId: string, requested: string | null, allowLocal: boolean): Promise { + const environments = await api.get(`/api/companies/${companyId}/environments`); + if (requested) { + if (!environments.some(environment => environment.id === requested)) throw new SetupError(`Environment ${requested} is not in the company`); + return requested; + } + const active = environments.filter(environment => environment.status !== "archived" && (allowLocal || environment.driver !== "local")); + const sandboxes = active.filter(environment => environment.driver === "sandbox"); + const candidates = sandboxes.length ? sandboxes : active; + if (candidates.length !== 1) { + throw new SetupError(`The company has ${candidates.length} usable environments; pass --environment `); + } + return String(candidates[0]!.id); +} + +export const AGENT_INSTRUCTIONS = [ + "You are a load-test worker for a Paperclip scale ramp.", + "Follow the assigned task literally and in order. Keep every reply short.", + "Use only the runtime's real tool-call channel for tools. Never print tool-call markup as text.", + "Never print, store or expose credential values. Never create other tasks, documents or agents.", +].join("\n"); + +export function agentPayload(input: { label: string; index: number; environmentId: string; profile: AgentProfile; secretId: string | null }): Json { + const { profile } = input; + const env = profile.credentialKey && input.secretId + ? { [profile.credentialKey]: { type: "secret_ref", secretId: input.secretId, version: "latest" } } + : {}; + return { + name: agentName(input.label, input.index), + role: "qa", + title: "Scale ramp worker", + capabilities: "Runs one small deterministic task at a time for a load test.", + adapterType: "paperclip_runner", + adapterConfig: { + provider: profile.provider, + model: profile.model, + lifecycleMode: "warm", + idleTimeoutMs: 300_000, + ...(profile.provider === "opencode" ? { opencodePermissionMode: "allow" } : {}), + env, + ...profile.adapterOverrides, + }, + defaultEnvironmentId: input.environmentId, + budgetMonthlyCents: 0, + runtimeConfig: { heartbeat: { enabled: false, maxConcurrentRuns: 1 } }, + instructionsBundle: { entryFile: "AGENTS.md", files: { "AGENTS.md": AGENT_INSTRUCTIONS } }, + }; +} + +export async function prepareCompany(input: { + api: PaperclipClient; + options: Options; + state: RunState; + store: StateStore; + credential: string | null; + log: (line: string) => void; +}): Promise { + const { api, options, state, store } = input; + await resolveCompany(api, options, state, store); + const companyId = state.companyId!; + state.environmentId = await pickEnvironment(api, companyId, options.environmentId, options.target.kind === "local"); + await store.save(state); + const profile = options.profile; + if (profile.credentialKey && !profile.providerSecretId) { + if (!input.credential) throw new SetupError(`Set ${profile.credentialKey} in the environment, or PAPERCLIP_SCALE_PROVIDER_SECRET_ID to reuse a stored secret`); + const secret = await api.post<{ id: string }>(`/api/companies/${companyId}/secrets`, { + name: `${secretPrefix(options.label)}${profile.credentialKey}`, + value: input.credential, + description: `Scale ramp ${options.label}; deleted at cleanup`, + }); + state.secretId = String(secret.id); + await store.save(state); + } + const project = await api.post<{ id: string }>(`/api/companies/${companyId}/projects`, { + name: projectName(options.label), + description: `Scale ramp ${options.label}. Created and archived by scripts/scale-agents-ramp.ts.`, + }); + state.projectId = String(project.id); + await store.save(state); + input.log(`Prepared company ${companyId}: environment ${state.environmentId}, project ${state.projectId}`); +} + +export async function ensureAgents(input: { + api: PaperclipClient; + options: Options; + state: RunState; + store: StateStore; + count: number; +}): Promise { + const { api, options, state, store } = input; + const secretId = options.profile.providerSecretId ?? state.secretId; + const start = state.agentIds.length; + const indices = Array.from({ length: Math.max(0, input.count - start) }, (_, offset) => start + offset); + await mapLimit(indices, AGENT_CREATE_PARALLELISM, async index => { + const created = await api.post<{ id: string }>(`/api/companies/${state.companyId}/agents`, agentPayload({ + label: options.label, index, environmentId: state.environmentId!, profile: options.profile, secretId, + })); + state.agentIds.push(String(created.id)); + await store.save(state); + }); +} + +export const AGENT_CREATE_PARALLELISM = 8; diff --git a/scripts/scale-ramp/state.ts b/scripts/scale-ramp/state.ts new file mode 100644 index 00000000000..8d7c6df7288 --- /dev/null +++ b/scripts/scale-ramp/state.ts @@ -0,0 +1,93 @@ +import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; +import path from "node:path"; + +export interface RunState { + schema: "paperclip.scale-ramp.state/v1"; + label: string; + baseUrl: string; + companyId: string | null; + companyCreated: boolean; + environmentId: string | null; + projectId: string | null; + secretId: string | null; + agentIds: string[]; + taskIds: string[]; + createdAt: string; +} + +export const companyName = (label: string) => `Scale ramp ${label}`; +export const projectName = (label: string) => `Scale ramp ${label}`; +export const agentPrefix = (label: string) => `SR ${label} `; +export const agentName = (label: string, index: number) => `${agentPrefix(label)}${String(index + 1).padStart(4, "0")}`; +export const secretPrefix = (label: string) => `scale-ramp ${label} `; +export const taskTag = (label: string) => `[${label}]`; + +export const statePath = (dir: string) => path.join(dir, "state.json"); + +export function emptyState(label: string, baseUrl: string, now: number): RunState { + return { + schema: "paperclip.scale-ramp.state/v1", label, baseUrl, companyId: null, companyCreated: false, environmentId: null, + projectId: null, secretId: null, agentIds: [], taskIds: [], createdAt: new Date(now).toISOString(), + }; +} + +export interface StateStore { + save(state: RunState): Promise; + load(): Promise; +} + +export function fileStateStore(dir: string): StateStore { + let queue: Promise = Promise.resolve(); + const write = async (body: string) => { + await mkdir(dir, { recursive: true, mode: 0o700 }); + const temporary = `${statePath(dir)}.${process.pid}.tmp`; + await writeFile(temporary, body, { mode: 0o600 }); + await rename(temporary, statePath(dir)); + }; + return { + save(state) { + const body = `${JSON.stringify(state, null, 2)}\n`; + const next = queue.then(() => write(body)); + queue = next.catch(() => undefined); + return next; + }, + async load() { + try { + return JSON.parse(await readFile(statePath(dir), "utf8")) as RunState; + } catch { + return null; + } + }, + }; +} + +export function memoryStateStore(initial: RunState | null = null): StateStore & { current: RunState | null } { + const store = { + current: initial ? structuredClone(initial) : null, + async save(state: RunState) { + store.current = structuredClone(state); + }, + async load() { + return store.current ? structuredClone(store.current) : null; + }, + }; + return store; +} + +export async function mapLimit(items: readonly T[], limit: number, run: (item: T) => Promise): Promise { + let next = 0; + let failure: { error: unknown } | null = null; + const workers = Array.from({ length: Math.min(limit, items.length) }, async () => { + while (!failure && next < items.length) { + const item = items[next]!; + next += 1; + try { + await run(item); + } catch (error) { + failure ??= { error }; + } + } + }); + await Promise.all(workers); + if (failure) throw (failure as { error: unknown }).error; +} diff --git a/scripts/scale-ramp/stats.ts b/scripts/scale-ramp/stats.ts new file mode 100644 index 00000000000..8ef24dfeb89 --- /dev/null +++ b/scripts/scale-ramp/stats.ts @@ -0,0 +1,26 @@ +export interface Distribution { + n: number; + p50: number | null; + p90: number | null; + p95: number | null; + p99: number | null; + max: number | null; +} + +export function percentile(sorted: readonly number[], p: number): number | null { + if (!sorted.length) return null; + const rank = Math.ceil((p / 100) * sorted.length) - 1; + return sorted[Math.min(sorted.length - 1, Math.max(0, rank))]!; +} + +export function distribution(values: readonly number[]): Distribution { + const sorted = values.filter(value => Number.isFinite(value)).sort((a, b) => a - b); + return { + n: sorted.length, + p50: percentile(sorted, 50), + p90: percentile(sorted, 90), + p95: percentile(sorted, 95), + p99: percentile(sorted, 99), + max: sorted.length ? sorted[sorted.length - 1]! : null, + }; +} diff --git a/scripts/scale-ramp/tsconfig.json b/scripts/scale-ramp/tsconfig.json new file mode 100644 index 00000000000..9d9c1c383d4 --- /dev/null +++ b/scripts/scale-ramp/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "noEmit": true, + "rootDir": "../..", + "allowImportingTsExtensions": true, + "erasableSyntaxOnly": true, + "verbatimModuleSyntax": true, + "types": ["node"], + "typeRoots": ["../../cli/node_modules/@types"] + }, + "include": ["./**/*.ts", "../scale-agents-ramp.ts"] +} diff --git a/scripts/scale-ramp/workload.ts b/scripts/scale-ramp/workload.ts new file mode 100644 index 00000000000..88267bc120c --- /dev/null +++ b/scripts/scale-ramp/workload.ts @@ -0,0 +1,69 @@ +import { createHash } from "node:crypto"; + +export interface RunSpan { + name: string; + startOffsetMs: number; + durationMs: number; +} + +export interface RunEvent { + eventType: string; + createdAt: string; + payload?: Record | null; +} + +export function taskNonce(label: string, index: number): string { + return createHash("sha256").update(`${label}:${index}`).digest("hex").slice(0, 10); +} + +export function taskTitle(tag: string, nonce: string): string { + return `${tag} Scale ramp task ${nonce}`; +} + +export function taskBrief(nonce: string): string { + return [ + `Scale ramp task ${nonce}. Do these steps in order and nothing else:`, + `1. Write one short sentence saying you are starting task ${nonce}.`, + "2. Run the shell command `pwd`, then run `ls`.", + `3. Write a file named \`scale-${nonce}.txt\` in the workspace root containing the single line \`${nonce}\`.`, + `4. Run the shell command \`cat scale-${nonce}.txt\`.`, + `5. Post a comment on this task that says \`CHECKPOINT ${nonce}\`.`, + `6. Post a final comment that says \`DONE ${nonce}\`, then mark this task done.`, + "Do not create other tasks, documents or files.", + ].join("\n"); +} + +function spans(events: readonly RunEvent[]): RunSpan[] { + const out: RunSpan[] = []; + for (const event of events) { + if (event.eventType !== "run.performance.span" || !event.payload) continue; + const { span, startOffsetMs, durationMs } = event.payload as { span?: unknown; startOffsetMs?: unknown; durationMs?: unknown }; + if (typeof span === "string" && typeof startOffsetMs === "number" && typeof durationMs === "number") out.push({ name: span, startOffsetMs, durationMs }); + } + return out; +} + +export interface RunTiming { + ttftMs: number | null; + firstActivityMs: number | null; +} + +const ACTIVITY_EVENTS = new Set(["item.delta", "tool.execution.started", "item.started"]); + +export function runTiming(events: readonly RunEvent[], runCreatedAt: number | null): RunTiming { + const all = spans(events); + const first = (name: string) => all.find(span => span.name === name) ?? null; + const queue = first("heartbeat.queue"); + const fromRunCreated = (span: RunSpan) => queue ? Math.max(0, span.startOffsetMs + span.durationMs - queue.startOffsetMs) : span.durationMs; + const fallback = (types: ReadonlySet) => { + if (runCreatedAt === null) return null; + const at = events.filter(event => types.has(event.eventType)).map(event => Date.parse(event.createdAt)).filter(Number.isFinite); + return at.length ? Math.max(0, Math.min(...at) - runCreatedAt) : null; + }; + const ttft = first("task.time_to_first_token"); + const activity = first("task.time_to_first_agent_activity"); + return { + ttftMs: ttft ? fromRunCreated(ttft) : fallback(new Set(["item.delta"])), + firstActivityMs: activity ? fromRunCreated(activity) : fallback(ACTIVITY_EVENTS), + }; +} diff --git a/scripts/typecheck-test-programs.mjs b/scripts/typecheck-test-programs.mjs index 0cb114d274c..edddf04d3ce 100644 --- a/scripts/typecheck-test-programs.mjs +++ b/scripts/typecheck-test-programs.mjs @@ -52,6 +52,7 @@ export const PROGRAMS = [ "packages/plugins/plugin-workspace-diff/tsconfig.test.json", "packages/plugins/sdk/tsconfig.test.json", ...SANDBOX_PROVIDERS.map((directory) => `${directory}/tsconfig.test.json`), + "scripts/scale-ramp/tsconfig.json", "tests/runner-acceptance/tsconfig.json", "tests/tsconfig.playwright.json", ]; diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index f3731c4042b..cc252e3f1b7 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -1,6 +1,6 @@ { "inputs": { - "package.json": "590fd8f555086a3bdc4dfa41a8016a89be13b977", + "package.json": "2d2a4abebb6a5931be1a8f0c8fb2615890324d21", "packages/plugins/sdk/package.json": "b7a4936e2803fd0d3b32dd7f17fc1ea86609ead0", "packages/plugins/sdk/src/bundlers.ts": "883bf3d9f2bd42877cfc0c3707cd882a6f6248d0", "packages/plugins/sdk/src/define-plugin.ts": "a99e07ce77ef72fcf1df1347346aacb8ba12e284",