diff --git a/doc/plans/2026-11-13-thousand-concurrent-agents.md b/doc/plans/2026-11-13-thousand-concurrent-agents.md index 24e850f2a4b..5816f6f4545 100644 --- a/doc/plans/2026-11-13-thousand-concurrent-agents.md +++ b/doc/plans/2026-11-13-thousand-concurrent-agents.md @@ -432,7 +432,12 @@ the one origin. above `--spend-threshold-usd` (default $5) unless `--confirm-spend ` is at least the estimate. It never creates more tasks than the printed cap, and it stops when the model spend the runs record passes the confirmed -amount. The defaults are assumptions: $0.02 model spend per task and $0.10 +amount. It reads recorded spend from Paperclip's cost records for the run's +agents (`/api/companies//costs/by-agent?period=all`); run rows carry no +cost. When some cost events have no known price, it prints a warning, grades +the run incomplete, and checks the cap against tasks created times the +per-task assumption as well. The +defaults are assumptions: $0.02 model spend per task and $0.10 per sandbox-hour (`PAPERCLIP_SCALE_MODEL_USD_PER_TASK`, `PAPERCLIP_SCALE_SANDBOX_USD_PER_HOUR`). Correct them from the recorded spend of the first run. @@ -451,7 +456,8 @@ A missing source is reported as "not measured", never as zero. | Criterion | Source | Target | Measured from | | --- | --- | --- | --- | | Runs that need Retry | finish line | 0, stops at once | task view offers Retry | -| Failed or timed-out runs | harness | 0, stops at once | run list | +| Failed runs whose task did not finish by itself | harness | 0 within `--recovery-grace-minutes` (default 15), stops then | run list and task status | +| Failed runs, recovered ones included, as a share of ended runs | harness | at most `--max-failed-run-percent` (default 2) | run list | | Tasks at rest but not done | harness | 0 | task status | | Peak running agents | harness | at least 90% of the step | run list | | Board page API requests, p90 | finish line | below 500 ms | a GET mix every 10 s: dashboard, issues, agents, live runs, sidebar badges | @@ -464,6 +470,23 @@ A missing source is reported as "not measured", never as zero. | Lease renewals that miss their deadline | finish line | 0 | metrics endpoint | | Kill one server process | finish line | runs resume, no Retry | not measured by this harness | +**Failed runs.** Paperclip retries many failed runs by itself (a same-run +bootstrap retry, a continuation run). A failed run does not stop the ramp +while its task can still finish: the harness keeps going and counts the run as +recovered when its task reaches `done`, and as unrecovered when the task comes +to rest without being done, offers Retry with no automatic retry scheduled, or +has no live run for the recovery grace after the failure. An unrecovered run +stops the ramp at once. The share of failed runs, recovered ones included, is +graded at the end of each step. + +**Cleanup.** The harness reads each agent by id (a terminated agent leaves the +company's agent list) and deletes it only after Paperclip reports its +termination complete (`lifecycleState` `terminated`). An agent Paperclip keeps +for its cost history stays terminated. The result names the agents still +terminating and the sandbox leases still active after the waits, and counts the +leases that released while it waited; `--cleanup-only ` finishes them +later from the state file. + **Server metrics.** Each server process can serve `GET /api/instance/server-metrics`, a JSON snapshot of that process over the last 50 to 60 s (`windowMs`), with its `processId` (random per process diff --git a/package.json b/package.json index 2d2a4abebb6..b6a185b015b 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "smoke:notion-generic-live": "node scripts/smoke/notion-generic-live.mjs", "smoke:posthog-live": "node scripts/smoke/posthog-live.mjs", "smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs", - "test:release-registry": "node scripts/ci-profile/node-test.mjs scripts/ci-profile/reporters.test.mjs scripts/ci/retry-reporter.test.mjs scripts/__tests__/runner-vitest-package-lanes.test.mjs scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/public-install-closure.test.mjs scripts/__tests__/public-install-registry.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/__tests__/runner-release-crate-cache.test.mjs scripts/__tests__/runner-debug-crate-cache.test.mjs scripts/__tests__/agent-build-cache.test.mjs scripts/__tests__/agent-preflight.test.mjs scripts/__tests__/vitest-worker-isolation.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/draft-stable-notes.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs scripts/typecheck-test-programs.test.mjs scripts/__tests__/local-verify.test.mjs scripts/test-sandbox-providers.test.mjs scripts/__tests__/ci-typecheck-lane.test.mjs scripts/scale-ramp/scale-ramp.test.ts", + "test:release-registry": "node scripts/ci-profile/node-test.mjs scripts/ci-profile/reporters.test.mjs scripts/ci/retry-reporter.test.mjs scripts/__tests__/runner-vitest-package-lanes.test.mjs scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/public-install-closure.test.mjs scripts/__tests__/public-install-registry.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/__tests__/runner-release-crate-cache.test.mjs scripts/__tests__/runner-debug-crate-cache.test.mjs scripts/__tests__/agent-build-cache.test.mjs scripts/__tests__/agent-preflight.test.mjs scripts/__tests__/vitest-worker-isolation.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/draft-stable-notes.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs scripts/typecheck-test-programs.test.mjs scripts/__tests__/local-verify.test.mjs scripts/test-sandbox-providers.test.mjs scripts/__tests__/ci-typecheck-lane.test.mjs scripts/scale-ramp/scale-ramp.test.ts scripts/scale-ramp/scale-ramp-spend.test.ts", "storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs", "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts", "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack", diff --git a/scripts/scale-ramp/cleanup.ts b/scripts/scale-ramp/cleanup.ts index f40b169ca97..bda4d807768 100644 --- a/scripts/scale-ramp/cleanup.ts +++ b/scripts/scale-ramp/cleanup.ts @@ -10,16 +10,23 @@ export interface CleanupReport { agentsTerminated: number; agentsDeleted: number; agentsKeptForCostHistory: number; + agentsStillTerminating: string[]; secretDeleted: boolean; projectArchived: boolean; companyArchived: boolean; activeSandboxLeases: number | null; + activeSandboxLeaseIds: string[]; + sandboxLeasesReleasedDuringWait: number; durationMs: number; errors: string[]; } export const LIVE_RUN_STATUSES = new Set(["queued", "running", "scheduled_retry"]); const PARALLELISM = 8; +const TERMINATED = new Set(["terminated", "rejected"]); +const TERMINATION_READS_PER_POLL = 200; +const TERMINATION_POLL_MS_PER_AGENT = 30; +const TERMINATION_POLL_MAX_MS = 30_000; export interface CleanupInput { api: PaperclipClient; @@ -53,8 +60,9 @@ export async function cleanupRun(input: CleanupInput): Promise { const started = clock.now(); const pollMs = input.pollMs ?? 3_000; const report: CleanupReport = { - agentsPaused: 0, runsCancelled: 0, tasksArchived: 0, agentsTerminated: 0, agentsDeleted: 0, agentsKeptForCostHistory: 0, - secretDeleted: false, projectArchived: false, companyArchived: false, activeSandboxLeases: null, durationMs: 0, errors: [], + agentsPaused: 0, runsCancelled: 0, tasksArchived: 0, agentsTerminated: 0, agentsDeleted: 0, agentsKeptForCostHistory: 0, agentsStillTerminating: [], + secretDeleted: false, projectArchived: false, companyArchived: false, activeSandboxLeases: null, activeSandboxLeaseIds: [], + sandboxLeasesReleasedDuringWait: 0, durationMs: 0, errors: [], }; const attempt = async (what: string, action: () => Promise): Promise => { try { @@ -79,8 +87,22 @@ export async function cleanupRun(input: CleanupInput): Promise { return report; } const prefix = agentPrefix(label); + const readAgent = async (id: string): Promise => { + try { + return await api.get(`/api/agents/${id}`); + } catch (error) { + if (error instanceof ApiError && error.status === 404) return null; + report.errors.push(`read agent ${id}: ${error instanceof Error ? error.message : String(error)}`.slice(0, 300)); + return { id, status: "unknown", lifecycleState: "unknown" }; + } + }; const listed = await read("list agents", `/api/companies/${companyId}/agents`, []); const agents = listed.filter(agent => String(agent.name ?? "").startsWith(prefix)); + const unlisted = state.agentIds.filter(id => !agents.some(agent => String(agent.id) === id)); + await mapLimit(unlisted, PARALLELISM, async id => { + const agent = await readAgent(id); + if (agent && String(agent.name ?? prefix).startsWith(prefix)) agents.push(agent); + }); const agentIds = new Set(agents.map(agent => String(agent.id))); input.log?.(`Cleanup ${label}: ${agents.length} agents, ${state.taskIds.length} recorded tasks`); @@ -121,16 +143,27 @@ export async function cleanupRun(input: CleanupInput): Promise { if (await attempt(`terminate agent ${agent.id}`, () => api.post(`/api/agents/${agent.id}/terminate`))) report.agentsTerminated += 1; }); const pending = new Set(agentIds); - for (const until = clock.now() + (input.terminateWaitMs ?? 300_000); pending.size && clock.now() < until;) { - const current = await read("list agents", `/api/companies/${companyId}/agents`, []); - for (const agent of current) { - if (!pending.has(String(agent.id))) continue; - if (agent.status === "terminated" && (agent.lifecycleState == null || agent.lifecycleState === "terminated")) pending.delete(String(agent.id)); - } - for (const id of [...pending]) if (!current.some(agent => String(agent.id) === id)) pending.delete(id); - if (pending.size) await clock.sleep(pollMs); + const waitMs = input.terminateWaitMs ?? 300_000; + const readPending = async (ids: string[]) => mapLimit(ids, PARALLELISM, async id => { + const agent = await readAgent(id); + if (!agent || TERMINATED.has(String(agent.lifecycleState))) pending.delete(id); + }); + let cursor = 0; + for (const until = clock.now() + waitMs; pending.size && clock.now() < until;) { + const ids = [...pending]; + const start = cursor % ids.length; + const batch = [...ids.slice(start), ...ids.slice(0, start)].slice(0, TERMINATION_READS_PER_POLL); + cursor = start + batch.length; + await readPending(batch); + if (!pending.size) break; + await clock.sleep(Math.min(TERMINATION_POLL_MAX_MS, Math.max(pollMs, pending.size * TERMINATION_POLL_MS_PER_AGENT))); + } + if (pending.size) await readPending([...pending]); + report.agentsStillTerminating = [...pending]; + if (pending.size) { + report.errors.push(`${pending.size} agents had not completed termination after ${Math.round(waitMs / 60_000)} min, so they were not deleted: ` + + `${[...pending].join(", ")}; run --cleanup-only ${label} again later`); } - if (pending.size) report.errors.push(`${pending.size} agents were not terminated in time; run --cleanup-only ${label} again`); await mapLimit([...agentIds].filter(id => !pending.has(id)), PARALLELISM, async agentId => { try { await api.delete(`/api/agents/${agentId}`); @@ -154,7 +187,10 @@ export async function cleanupRun(input: CleanupInput): Promise { } if (state.environmentId && taskIds.size) { - for (const until = clock.now() + (input.leaseWaitMs ?? 300_000); ;) { + const leaseWaitMs = input.leaseWaitMs ?? 300_000; + const seen = new Set(); + let active: Json[] = []; + for (const until = clock.now() + leaseWaitMs; ;) { let leases: Json[] | null = null; try { leases = await api.get(`/api/environments/${state.environmentId}/leases?status=active`); @@ -162,11 +198,18 @@ export async function cleanupRun(input: CleanupInput): Promise { leases = null; } if (leases === null) break; - report.activeSandboxLeases = leases.filter(lease => taskIds.has(String(lease.issueId))).length; - if (report.activeSandboxLeases === 0 || clock.now() >= until) break; + active = leases.filter(lease => taskIds.has(String(lease.issueId)) || agentIds.has(String(lease.metadata?.agentId ?? ""))); + for (const lease of active) seen.add(String(lease.id)); + report.activeSandboxLeases = active.length; + if (active.length === 0 || clock.now() >= until) break; await clock.sleep(pollMs); } - if (report.activeSandboxLeases) report.errors.push(`${report.activeSandboxLeases} sandbox leases of this run are still active`); + report.activeSandboxLeaseIds = active.map(lease => String(lease.id)); + report.sandboxLeasesReleasedDuringWait = [...seen].filter(id => !report.activeSandboxLeaseIds.includes(id)).length; + if (active.length) { + const detail = active.map(lease => `${lease.id} (run ${lease.heartbeatRunId ?? "unknown"}, acquired ${lease.acquiredAt ?? "unknown"})`).join(", "); + report.errors.push(`${active.length} sandbox leases of this run were still active after ${Math.round(leaseWaitMs / 60_000)} min: ${detail}`.slice(0, 600)); + } } if (state.companyCreated) { diff --git a/scripts/scale-ramp/config.ts b/scripts/scale-ramp/config.ts index 65db8760d38..9ac7bfe4940 100644 --- a/scripts/scale-ramp/config.ts +++ b/scripts/scale-ramp/config.ts @@ -21,6 +21,8 @@ export interface Thresholds { queueWaitP95Ms: number; firstActivityP95Ms: number; minConcurrencyRatio: number; + maxFailedRunRatio: number; + recoveryGraceMs: number; maxEventLoopUtilization: number; runEventLatenessP99Ms: number; agentApiPreHandlerP99Ms: number; @@ -159,6 +161,8 @@ export function parseOptions(argv: readonly string[], env: NodeJS.ProcessEnv, no "max-queue-wait-p95-seconds": { type: "string" }, "max-first-activity-p95-seconds": { type: "string" }, "min-concurrency-ratio": { type: "string" }, + "max-failed-run-percent": { type: "string" }, + "recovery-grace-minutes": { type: "string" }, }, }); if (values["dry-run"] && values["cleanup-only"]) throw new Error("--dry-run and --cleanup-only exclude each other"); @@ -198,6 +202,8 @@ export function parseOptions(argv: readonly string[], env: NodeJS.ProcessEnv, no queueWaitP95Ms: number(values["max-queue-wait-p95-seconds"], "--max-queue-wait-p95-seconds", 60, 1, 3_600) * 1_000, firstActivityP95Ms: number(values["max-first-activity-p95-seconds"], "--max-first-activity-p95-seconds", 60, 1, 3_600) * 1_000, minConcurrencyRatio: number(values["min-concurrency-ratio"], "--min-concurrency-ratio", 0.9, 0.1, 1), + maxFailedRunRatio: number(values["max-failed-run-percent"], "--max-failed-run-percent", 2, 0, 100) / 100, + recoveryGraceMs: number(values["recovery-grace-minutes"], "--recovery-grace-minutes", 15, 1, 120) * 60_000, maxEventLoopUtilization: 0.6, runEventLatenessP99Ms: 1_000, agentApiPreHandlerP99Ms: 100, diff --git a/scripts/scale-ramp/criteria.ts b/scripts/scale-ramp/criteria.ts index be596ff392e..32691fe6eb6 100644 --- a/scripts/scale-ramp/criteria.ts +++ b/scripts/scale-ramp/criteria.ts @@ -22,7 +22,11 @@ export interface StepMetrics { tasksFinished: number; tasksFailed: number; tasksNeedingRetry: number; + runsEnded: number; runsFailed: number; + runsFailedRecovered: number; + runsFailedUnrecovered: number; + runsFailedPending: number; runsInterrupted: number; runsCancelled: number; peakRunning: number; @@ -32,7 +36,7 @@ export interface StepMetrics { ttftMs: Distribution; firstActivityMs: Distribution; runDurationMs: Distribution; - board: { requests: number; failures: number; ms: Distribution }; + board: { requests: number; failures: number; ms: Distribution; byRoute: Record }; server: ServerMetrics; recordedSpendUsd: number; } @@ -69,10 +73,17 @@ export const CRITERIA: readonly Criterion[] = [ evaluate: m => ({ status: m.tasksNeedingRetry === 0 ? "pass" : "fail", observed: m.tasksNeedingRetry }), }, { - id: "failed_runs_zero", source: "harness", immediate: true, - description: "Runs that ended failed or timed out", - threshold: () => "= 0", - evaluate: m => ({ status: m.runsFailed === 0 ? "pass" : "fail", observed: m.runsFailed }), + id: "failed_runs_unrecovered_zero", source: "harness", immediate: true, + description: "Failed or timed-out runs whose task did not finish by itself within the recovery grace", + threshold: t => `= 0 (grace ${Math.round(t.recoveryGraceMs / 60_000)} min)`, + evaluate: m => ({ status: m.runsFailedUnrecovered === 0 ? "pass" : "fail", observed: m.runsFailedUnrecovered }), + }, + { + id: "failed_run_rate", source: "harness", immediate: false, + description: "Failed or timed-out runs as a share of ended runs, recovered ones included (%)", + threshold: t => `<= ${Math.round(t.maxFailedRunRatio * 1_000) / 10}%`, + evaluate: (m, t) => m.runsEnded === 0 ? { status: "not_measured", observed: null } + : { status: m.runsFailed / m.runsEnded <= t.maxFailedRunRatio ? "pass" : "fail", observed: Math.round((m.runsFailed / m.runsEnded) * 1_000) / 10 }, }, { id: "tasks_rested_not_done_zero", source: "harness", immediate: false, diff --git a/scripts/scale-ramp/fake-paperclip.ts b/scripts/scale-ramp/fake-paperclip.ts index 0dcddcac131..eb087f2e200 100644 --- a/scripts/scale-ramp/fake-paperclip.ts +++ b/scripts/scale-ramp/fake-paperclip.ts @@ -25,7 +25,14 @@ export interface FakeConfig { runDurationMs: number; capacity: number; failWhenRunningAbove: number; + failRun: (startIndex: number) => boolean; retryOnFailure: boolean; + recoverFailedRunAfterMs: number | null; + retrySameRunAfterMs: number | null; + sameRunRetryDurationMs: number | null; + unpricedEventsPerAgent: number; + terminationDelayMs: number; + leaseLingerMs: number; costPerRunUsd: number; boardLatencyMs: (running: number) => number; failRequest: (method: string, path: string, count: number) => { status: number; body: string } | null; @@ -42,6 +49,8 @@ interface FakeRun { startedAt: number | null; finishedAt: number | null; willFail: boolean; + recovery: boolean; + retryAt?: number | null; } export interface RequestLog { @@ -66,6 +75,7 @@ export class FakePaperclip { readonly environmentId = "00000000-0000-4000-8000-0000000000e1"; private seq = 0; private agentsCreated = 0; + private runsStarted = 0; private counts = new Map(); readonly clock: VirtualClock; @@ -82,7 +92,14 @@ export class FakePaperclip { runDurationMs: 60_000, capacity: Number.POSITIVE_INFINITY, failWhenRunningAbove: Number.POSITIVE_INFINITY, + failRun: () => false, retryOnFailure: false, + recoverFailedRunAfterMs: null, + retrySameRunAfterMs: null, + sameRunRetryDurationMs: null, + unpricedEventsPerAgent: 0, + terminationDelayMs: 0, + leaseLingerMs: 0, costPerRunUsd: 0.01, boardLatencyMs: () => 40, failRequest: () => null, @@ -108,7 +125,7 @@ export class FakePaperclip { for (let index = 0; index < count; index += 1) { const id = this.id(); this.runs.set(id, { id, agentId: "noise-agent", issueId: "noise-issue", status: "succeeded", createdAt: this.clock.now() + 1e10 + index, - startedAt: null, finishedAt: null, willFail: false }); + startedAt: null, finishedAt: null, willFail: false, recovery: false }); } } @@ -128,6 +145,7 @@ export class FakePaperclip { advance(): void { const now = this.clock.now(); for (const agent of this.agents.values()) { + if (agent.lifecycleState === "terminating" && now >= agent.terminatingSince + this.config.terminationDelayMs) agent.lifecycleState = "terminated"; const delay = typeof this.config.agentReadyDelayMs === "function" ? this.config.agentReadyDelayMs(agent.index) : this.config.agentReadyDelayMs; if (!["preparing", "verifying"].includes(agent.lifecycleState) || now < agent.createdAt + delay) continue; if (this.config.agentSetupError) { @@ -138,16 +156,38 @@ export class FakePaperclip { agent.status = "idle"; } } + for (const run of this.runs.values()) { + if (run.status !== "failed" || !run.retryAt || now < run.retryAt) continue; + run.status = "running"; + run.startedAt = run.retryAt; + run.finishedAt = null; + run.retryAt = null; + run.willFail = false; + run.recovery = true; + } for (const run of [...this.runs.values()].sort((a, b) => a.createdAt - b.createdAt)) { - if (run.status === "running" && run.startedAt !== null && now >= run.startedAt + this.config.runDurationMs) { - run.finishedAt = run.startedAt + this.config.runDurationMs; + const duration = run.recovery && this.config.sameRunRetryDurationMs !== null ? this.config.sameRunRetryDurationMs : this.config.runDurationMs; + if (run.status === "running" && run.startedAt !== null && now >= run.startedAt + duration) { + run.finishedAt = run.startedAt + duration; const issue = this.issues.get(run.issueId)!; if (run.willFail) { run.status = "failed"; issue.interruption = this.config.retryOnFailure ? { state: "failed", canRetry: true, retryWait: false } : null; + if (this.config.retrySameRunAfterMs !== null) { + run.retryAt = run.finishedAt + this.config.retrySameRunAfterMs; + issue.interruption = { state: "retry_scheduled", canRetry: true, nextRetryAt: new Date(run.retryAt).toISOString() }; + } + if (this.config.recoverFailedRunAfterMs !== null) { + const retryAt = run.finishedAt + this.config.recoverFailedRunAfterMs; + issue.interruption = { state: "retry_scheduled", canRetry: true, nextRetryAt: new Date(retryAt).toISOString() }; + const id = this.id(); + this.runs.set(id, { id, agentId: run.agentId, issueId: run.issueId, status: "queued", createdAt: retryAt, startedAt: null, finishedAt: null, + willFail: false, recovery: true }); + } } else { run.status = "succeeded"; issue.status = "done"; + issue.interruption = null; } } } @@ -156,7 +196,8 @@ export class FakePaperclip { if (this.running() >= this.config.capacity) break; run.status = "running"; run.startedAt = Math.max(run.createdAt + this.config.startDelayMs, now - 1_000); - run.willFail = this.running() > this.config.failWhenRunningAbove; + this.runsStarted += 1; + run.willFail = !run.recovery && (this.running() > this.config.failWhenRunningAbove || this.config.failRun(this.runsStarted)); this.issues.get(run.issueId)!.status = "in_progress"; } } @@ -165,8 +206,7 @@ export class FakePaperclip { const iso = (value: number | null) => value === null ? null : new Date(value).toISOString(); return { id: run.id, agentId: run.agentId, issueId: run.issueId, status: run.status, createdAt: iso(run.createdAt), startedAt: iso(run.startedAt), - finishedAt: iso(run.finishedAt), costUsd: run.status === "succeeded" || run.status === "failed" ? this.config.costPerRunUsd : null, - error: null, errorCode: null, + finishedAt: iso(run.finishedAt), costUsd: null, error: null, errorCode: null, }; } @@ -227,8 +267,18 @@ export class FakePaperclip { this.projects.set(id, { id, companyId, name: body.name, archivedAt: null }); return json(201, this.projects.get(id)); } + if (c === "costs" && parts[3] === "by-agent" && method === "GET") { + const cents = new Map(); + const month = new Date(now).toISOString().slice(0, 7); + for (const run of this.runs.values()) { + if (run.status !== "succeeded" && run.status !== "failed") continue; + if (query.get("period") !== "all" && new Date(run.finishedAt!).toISOString().slice(0, 7) !== month) continue; + cents.set(run.agentId, (cents.get(run.agentId) ?? 0) + this.config.costPerRunUsd * 100); + } + return json(200, [...cents].map(([agentId, costCents]) => ({ agentId, costCents, unpricedEventCount: this.config.unpricedEventsPerAgent }))); + } if (c === "agents") { - if (method === "GET") return json(200, [...this.agents.values()].filter(agent => agent.companyId === companyId)); + if (method === "GET") return json(200, [...this.agents.values()].filter(agent => agent.companyId === companyId && agent.status !== "terminated")); const id = this.id(); this.agents.set(id, { id, companyId, name: body.name, status: "paused", lifecycleState: "preparing", lifecycleHolds: [], lifecycleError: null, pauseReason: null, createdAt: now, index: this.agentsCreated++, payload: body }); @@ -242,7 +292,8 @@ export class FakePaperclip { const agent = this.agents.get(body.assigneeAgentId); if (agent && agent.status !== "terminated") { const runId = this.id(); - this.runs.set(runId, { id: runId, agentId: body.assigneeAgentId, issueId: id, status: "queued", createdAt: now, startedAt: null, finishedAt: null, willFail: false }); + this.runs.set(runId, { id: runId, agentId: body.assigneeAgentId, issueId: id, status: "queued", createdAt: now, startedAt: null, finishedAt: null, + willFail: false, recovery: false }); } return json(201, this.issues.get(id)); } @@ -267,16 +318,19 @@ export class FakePaperclip { if (a === "agents" && b) { const agent = this.agents.get(b); if (!agent) return json(404, { error: "Agent not found" }); + if (method === "GET" && !c) return json(200, agent); if (method === "PATCH") { Object.assign(agent, body); return json(200, agent); } if (method === "POST" && c === "terminate") { agent.status = "terminated"; - agent.lifecycleState = "terminated"; + agent.lifecycleState = this.config.terminationDelayMs > 0 ? "terminating" : "terminated"; + agent.terminatingSince = now; return json(200, agent); } if (method === "DELETE") { + if (agent.lifecycleState !== "terminated") return json(409, { error: "Complete termination before deleting the agent" }); this.agents.delete(b); return json(200, { ok: true }); } @@ -324,7 +378,10 @@ export class FakePaperclip { return json(200, project); } if (a === "environments" && b === this.environmentId && c === "leases") { - return json(200, this.liveRuns().map(run => ({ id: run.id, issueId: run.issueId, status: "active" }))); + const held = [...this.runs.values()].filter(run => ["queued", "running"].includes(run.status) + || (run.finishedAt !== null && now < run.finishedAt + this.config.leaseLingerMs)); + return json(200, held.map(run => ({ id: `lease-${run.id}`, issueId: run.issueId, heartbeatRunId: run.id, status: "active", + acquiredAt: new Date(run.createdAt).toISOString(), metadata: { agentId: run.agentId } }))); } return json(404, { error: `fake: no route for ${method} ${route}` }); } diff --git a/scripts/scale-ramp/ramp.ts b/scripts/scale-ramp/ramp.ts index 848ac9c267c..82206b718a8 100644 --- a/scripts/scale-ramp/ramp.ts +++ b/scripts/scale-ramp/ramp.ts @@ -1,9 +1,9 @@ -import { readMetrics, type Clock, type Fetcher, type PaperclipClient } from "./client.ts"; +import { pathClass, readMetrics, type Clock, type Fetcher, type PaperclipClient } from "./client.ts"; import type { Options, Step } from "./config.ts"; import { evaluate, firstFailure, type CriterionResult, type ServerMetrics, type StepMetrics } from "./criteria.ts"; import { LIVE_RUN_STATUSES } from "./cleanup.ts"; import { AgentsNotReady, describeAgent, ensureAgents, waitForAgentsReady } from "./setup.ts"; -import { distribution } from "./stats.ts"; +import { distribution, type Distribution } from "./stats.ts"; import { mapLimit, taskTag, type RunState, type StateStore } from "./state.ts"; import { runTiming, taskBrief, taskNonce, taskTitle, type RunEvent } from "./workload.ts"; @@ -27,6 +27,7 @@ const RETRY_RECHECK_MS = 30_000; const EVENT_FETCHES_PER_TICK = 10; const EVENT_PAGES = 4; const SINGLE_READS_PER_TICK = 50; +const SPEND_READ_FAILURES_TO_STOP = 3; const AGENT_RUN_PAGE = 20; const AGENT_RUN_READ_PARALLELISM = 8; @@ -50,7 +51,8 @@ interface RunRecord { createdAt: number | null; startedAt: number | null; finishedAt: number | null; - costUsd: number; + everFailed: boolean; + everEnded: boolean; sampled: boolean; error: string | null; errorCode: string | null; @@ -63,12 +65,14 @@ interface StepAccumulator { endedAt: number | null; runningSamples: number[]; boardMs: number[]; + boardByRoute: Record; boardRequests: number; boardFailures: number; serverSamples: Array>; ttftMs: number[]; firstActivityMs: number[]; sampled: number; + spendAtStartUsd: number; } export interface StepResult { @@ -93,7 +97,9 @@ export interface RampOutcome { openTasksAtEnd: number; tasksCreated: number; taskCapReached: boolean; - recordedSpendUsd: number; + recordedSpendUsd: number | null; + spendReadError: string | null; + unpricedCostEvents: number; runsListTruncated: boolean; metricsSource: "endpoint" | "not provided" | "unavailable"; } @@ -192,6 +198,10 @@ export class Ramp { private nextBoardAt = 0; private nextPollAt = 0; private taskCapReached = false; + private spendUsd: number | null = null; + private spendReadFailures = 0; + private spendReadError: string | null = null; + private unpricedCostEvents = 0; private fallbackCursor = 0; private runsListTruncated = false; private metricsSource: RampOutcome["metricsSource"]; @@ -220,8 +230,8 @@ export class Ramp { break; } log(`Step ${step.index + 1}/${options.steps.length}: ${step.concurrency} agents for ${Math.round(step.holdMs / 60_000)} min`); - const accumulator: StepAccumulator = { step, startedAt: clock.now(), endedAt: null, runningSamples: [], boardMs: [], boardRequests: 0, - boardFailures: 0, serverSamples: [], ttftMs: [], firstActivityMs: [], sampled: 0 }; + const accumulator: StepAccumulator = { step, startedAt: clock.now(), endedAt: null, runningSamples: [], boardMs: [], boardByRoute: {}, boardRequests: 0, + boardFailures: 0, serverSamples: [], ttftMs: [], firstActivityMs: [], sampled: 0, spendAtStartUsd: this.spendUsd ?? 0 }; this.accumulators.push(accumulator); stop = await this.prepareStep(step); if (!stop) { @@ -249,13 +259,15 @@ export class Ramp { const last = this.current(); last.endedAt = clock.now(); } + const finalSpendRead = await this.readSpend(); for (const result of results) { result.metrics = this.metrics(result.index); result.criteriaFinal = evaluate(result.metrics, options.thresholds); } return { steps: results, stop, drained, openTasksAtEnd: this.openTasks().length, tasksCreated: this.tasks.size, taskCapReached: this.taskCapReached, - recordedSpendUsd: this.recordedSpend(), runsListTruncated: this.runsListTruncated, metricsSource: this.metricsSource, + recordedSpendUsd: finalSpendRead ? this.spendUsd : null, spendReadError: finalSpendRead ? null : this.spendReadError, + unpricedCostEvents: this.unpricedCostEvents, runsListTruncated: this.runsListTruncated, metricsSource: this.metricsSource, }; } @@ -374,8 +386,17 @@ export class Ramp { } private immediateStop(stepIndex: number): StopReason | null { - if (this.recordedSpend() > this.deps.spendCapUsd) { - return { stepIndex, reason: `recorded model spend $${this.recordedSpend().toFixed(2)} passed the cap of $${this.deps.spendCapUsd}`, criterion: "spend_cap" }; + if ((this.spendUsd ?? 0) > this.deps.spendCapUsd) { + return { stepIndex, reason: `recorded model spend $${this.spendUsd!.toFixed(2)} passed the cap of $${this.deps.spendCapUsd}`, criterion: "spend_cap" }; + } + const bound = this.unpricedSpendBoundUsd(); + if (bound !== null && bound > this.deps.spendCapUsd) { + return { stepIndex, reason: `${this.unpricedCostEvents} cost events have no known price, so the spend cap uses tasks created x $${this.deps.options.pricing.modelUsdPerTask}` + + ` per task: $${bound.toFixed(2)} passed the cap of $${this.deps.spendCapUsd}`, criterion: "spend_cap" }; + } + if (this.spendReadFailures >= SPEND_READ_FAILURES_TO_STOP) { + return { stepIndex, reason: `recorded model spend could not be read ${this.spendReadFailures} times in a row, so the spend cap cannot be enforced: ${this.spendReadError}`, + criterion: "spend_unreadable" }; } for (const accumulator of this.accumulators) { const failed = firstFailure(evaluate(this.metrics(accumulator.step.index), this.deps.options.thresholds, true)); @@ -388,10 +409,39 @@ export class Ramp { return [...this.tasks.values()].filter(task => task.outcome === "open"); } - private recordedSpend(): number { - let total = 0; - for (const run of this.runs.values()) total += run.costUsd; - return Math.round(total * 100) / 100; + private unpricedSpendBoundUsd(): number | null { + if (this.unpricedCostEvents === 0) return null; + return Math.round(this.tasks.size * this.deps.options.pricing.modelUsdPerTask * 100) / 100; + } + + private async readSpend(): Promise { + const agentIds = new Set(this.deps.state.agentIds); + if (!agentIds.size) return this.spendReadFailures === 0; + let rows: unknown; + try { + rows = await this.deps.api.get(`/api/companies/${this.companyId}/costs/by-agent?period=all`); + if (!Array.isArray(rows)) throw new Error("costs by agent did not return a list"); + } catch (error) { + this.spendReadFailures += 1; + this.spendReadError = error instanceof Error ? error.message : String(error); + return false; + } + this.spendReadFailures = 0; + this.spendReadError = null; + let cents = 0; + let unpriced = 0; + for (const row of rows as Json[]) { + if (!agentIds.has(String(row.agentId))) continue; + cents += num(row.costCents) ?? 0; + unpriced += num(row.unpricedEventCount) ?? 0; + } + if (unpriced > 0 && this.unpricedCostEvents === 0) { + this.deps.log(`Warning: ${unpriced} cost events of this run have no known price. Recorded spend is a lower bound, so the spend cap now uses` + + ` tasks created x $${this.deps.options.pricing.modelUsdPerTask} per task (PAPERCLIP_SCALE_MODEL_USD_PER_TASK)`); + } + this.unpricedCostEvents = unpriced; + this.spendUsd = Math.round(cents) / 100; + return true; } private async topUp(concurrency: number, stepIndex: number, until: number): Promise { @@ -425,12 +475,18 @@ export class Ramp { const accumulator = this.current(); for (const route of BOARD_MIX) { const before = this.deps.api.records.length; - await this.deps.api.get(route(this.companyId), "board").catch(() => null); + const path = route(this.companyId); + await this.deps.api.get(path, "board").catch(() => null); const record = this.deps.api.records[before]; if (!record) continue; accumulator.boardRequests += 1; - if (record.ok) accumulator.boardMs.push(record.ms); - else accumulator.boardFailures += 1; + if (!record.ok) { + accumulator.boardFailures += 1; + continue; + } + accumulator.boardMs.push(record.ms); + const name = pathClass(path).split("/").at(-1)!; + (accumulator.boardByRoute[name] ??= []).push(record.ms); } } @@ -439,6 +495,7 @@ export class Ramp { await this.readTasks(); await this.checkRetry(); await this.sampleEvents(); + await this.readSpend(); let running = 0; for (const run of this.runs.values()) if (run.status === "running") running += 1; if (!recordSample) return; @@ -457,9 +514,11 @@ export class Ramp { if (!this.tasks.has(taskId)) return; const id = String(row.id ?? row.runId); const previous = this.runs.get(id); + const status = String(row.status); this.runs.set(id, { - id, taskId, agentId: String(row.agentId), status: String(row.status), createdAt: ms(row.createdAt), startedAt: ms(row.startedAt), - finishedAt: ms(row.finishedAt), costUsd: num(row.costUsd) ?? previous?.costUsd ?? 0, sampled: previous?.sampled ?? false, + id, taskId, agentId: String(row.agentId), status, createdAt: ms(row.createdAt), startedAt: ms(row.startedAt), + finishedAt: ms(row.finishedAt), everFailed: (previous?.everFailed ?? false) || FAILED_RUN.has(status), + everEnded: (previous?.everEnded ?? false) || ENDED_RUN.has(status), sampled: previous?.sampled ?? false, error: typeof row.error === "string" && row.error ? row.error : null, errorCode: typeof row.errorCode === "string" && row.errorCode ? row.errorCode : null, liveness: typeof row.livenessReason === "string" && row.livenessReason ? row.livenessReason : null, @@ -532,7 +591,9 @@ export class Ramp { if (task.outcome === "failed" && task.lastRetryCheckAt !== null) continue; task.lastRetryCheckAt = now; const view = await this.deps.api.get(`/api/issues/${task.id}`).catch(() => null); - if (view && view.status !== "done" && view.interruption?.canRetry === true && !view.interruption?.retryWait) { + const interruption = view?.interruption; + const systemRetrying = interruption?.state === "retry_scheduled" || Boolean(interruption?.nextRetryAt) || Boolean(interruption?.retryWait); + if (view && view.status !== "done" && interruption?.canRetry === true && !systemRetrying) { task.neededRetry = true; this.deps.log(`Task ${task.nonce} offers Retry`); } @@ -576,8 +637,10 @@ export class Ramp { if (known === undefined || run.startedAt < known) firstStart.set(run.taskId, run.startedAt); } const samples = accumulator.runningSamples; - let spend = 0; - for (const run of runs) spend += run.costUsd; + const failed = this.classifyFailedRuns(runs); + const spendAtEnd = this.accumulators[stepIndex + 1]?.spendAtStartUsd ?? this.spendUsd ?? 0; + const byRoute: Record = {}; + for (const [name, values] of Object.entries(accumulator.boardByRoute)) byRoute[name] = distribution(values); return { step: accumulator.step, startedAt: accumulator.startedAt, @@ -587,7 +650,11 @@ export class Ramp { tasksFinished: tasks.filter(task => task.outcome === "done").length, tasksFailed: tasks.filter(task => task.outcome === "failed").length, tasksNeedingRetry: tasks.filter(task => task.neededRetry).length, - runsFailed: runs.filter(run => FAILED_RUN.has(run.status)).length, + runsEnded: runs.filter(run => run.everEnded).length, + runsFailed: failed.recovered + failed.unrecovered + failed.pending, + runsFailedRecovered: failed.recovered, + runsFailedUnrecovered: failed.unrecovered, + runsFailedPending: failed.pending, runsInterrupted: runs.filter(run => run.status === "interrupted").length, runsCancelled: runs.filter(run => run.status === "cancelled").length, peakRunning: samples.length ? Math.max(...samples) : 0, @@ -597,12 +664,27 @@ export class Ramp { ttftMs: distribution(accumulator.ttftMs), firstActivityMs: distribution(accumulator.firstActivityMs), runDurationMs: distribution(runs.flatMap(run => run.startedAt !== null && run.finishedAt !== null ? [run.finishedAt - run.startedAt] : [])), - board: { requests: accumulator.boardRequests, failures: accumulator.boardFailures, ms: distribution(accumulator.boardMs) }, + board: { requests: accumulator.boardRequests, failures: accumulator.boardFailures, ms: distribution(accumulator.boardMs), byRoute }, server: serverMetrics(accumulator.serverSamples, { earlier: this.accumulators.slice(0, stepIndex).flatMap(earlier => earlier.serverSamples), runStartedAt: this.accumulators[0]?.startedAt ?? null, }), - recordedSpendUsd: Math.round(spend * 100) / 100, + recordedSpendUsd: Math.round(Math.max(0, spendAtEnd - accumulator.spendAtStartUsd) * 100) / 100, }; } + + private classifyFailedRuns(runs: readonly RunRecord[]): { recovered: number; unrecovered: number; pending: number } { + const now = this.deps.clock.now(); + const out = { recovered: 0, unrecovered: 0, pending: 0 }; + for (const run of runs) { + if (!run.everFailed) continue; + const task = this.tasks.get(run.taskId)!; + const live = this.runsOf(task.id).some(candidate => LIVE_RUN_STATUSES.has(candidate.status)); + if (task.outcome === "done") out.recovered += 1; + else if (task.outcome === "failed" || task.neededRetry) out.unrecovered += 1; + else if (!live && run.finishedAt !== null && now - run.finishedAt >= this.deps.options.thresholds.recoveryGraceMs) out.unrecovered += 1; + else out.pending += 1; + } + return out; + } } diff --git a/scripts/scale-ramp/report.ts b/scripts/scale-ramp/report.ts index 745dbc1e03a..f52679b8bf6 100644 --- a/scripts/scale-ramp/report.ts +++ b/scripts/scale-ramp/report.ts @@ -32,6 +32,7 @@ export interface RampResult { tasksCreated: number; taskCapReached: boolean; recordedSpendUsd: number | null; + unpricedCostEvents: number; sources: Record; cleanup: CleanupReport | null; requests: { total: number; failed: number }; @@ -53,6 +54,10 @@ export function incompleteReasons(outcome: RampOutcome | null): string[] { const missing = notMeasured(outcome); if (missing.length) reasons.push(`criteria not measured: ${missing.join(", ")}`); if (outcome.taskCapReached) reasons.push("the task cap was reached, so a step may have run below its concurrency"); + if (outcome.recordedSpendUsd === null) reasons.push("recorded model spend could not be read at the end of the run"); + if (outcome.unpricedCostEvents > 0) { + reasons.push(`${outcome.unpricedCostEvents} cost events have no known price, so recorded spend is a lower bound and the spend cap does not cover them`); + } if (outcome.openTasksAtEnd > 0) reasons.push(`${outcome.openTasksAtEnd} tasks were still open when the drain ended`); return reasons; } @@ -100,9 +105,13 @@ export function buildResult(input: { tasksCreated: outcome?.tasksCreated ?? 0, taskCapReached: outcome?.taskCapReached ?? false, recordedSpendUsd: outcome ? outcome.recordedSpendUsd : null, + unpricedCostEvents: outcome?.unpricedCostEvents ?? 0, sources: { runsAndTasks: outcome?.runsListTruncated ? "API (company run list was full; open tasks without a listed run were read one by one)" : "API", - boardLatency: "harness GET mix (dashboard, issues, agents, live runs, sidebar badges)", + boardLatency: "harness GET mix (dashboard, issues, agents, live runs, sidebar badges), with a distribution per route", + modelSpend: outcome?.spendReadError ? `not measured: the last read of costs by agent failed (${outcome.spendReadError})` + : `costs by agent for the run's agents over all time (Paperclip's recorded cost events)${outcome?.unpricedCostEvents + ? `; ${outcome.unpricedCostEvents} events have no known price and are not included` : ""}`, firstTokenAndActivity: `run events of up to ${options.eventSamplePerStep} succeeded runs per step`, serverMetrics: outcome?.metricsSource === "endpoint" ? "PAPERCLIP_SCALE_METRICS_URL" : outcome?.metricsSource === "unavailable" ? "PAPERCLIP_SCALE_METRICS_URL gave no readable sample: not measured" : "not provided: not measured", @@ -129,7 +138,8 @@ export function summaryLines(result: RampResult): string[] { "", `Verdict: **${result.verdict}**. Target ${result.target.baseUrl} (${result.target.kind}).`, result.stop ? `Stopped at step ${result.stop.stepIndex + 1}: ${result.stop.reason}.` : "No stop.", - `Tasks created: ${result.tasksCreated}${result.taskCapReached ? " (task cap reached)" : ""}. Recorded model spend: ${result.recordedSpendUsd === null ? "not measured" : `$${result.recordedSpendUsd}`}; estimate was $${result.plan.estimate.totalUsd}.`, + `Tasks created: ${result.tasksCreated}${result.taskCapReached ? " (task cap reached)" : ""}. Recorded model spend: ${result.recordedSpendUsd === null ? "not measured" : `$${result.recordedSpendUsd}`}` + + `${result.unpricedCostEvents ? ` (incomplete: ${result.unpricedCostEvents} cost events have no known price)` : ""}; estimate was $${result.plan.estimate.totalUsd}.`, "", ]; for (const step of result.steps) { @@ -138,13 +148,15 @@ export function summaryLines(result: RampResult): string[] { `## Step ${step.index + 1}: ${step.concurrency} agents, ${step.holdMs / 60_000} min`, "", `- Tasks created / started / finished / failed / needed Retry: ${m.tasksCreated} / ${m.tasksStarted} / ${m.tasksFinished} / ${m.tasksFailed} / ${m.tasksNeedingRetry}`, - `- Runs failed / interrupted / cancelled: ${m.runsFailed} / ${m.runsInterrupted} / ${m.runsCancelled}`, + `- Runs ended / failed / interrupted / cancelled: ${m.runsEnded} / ${m.runsFailed} / ${m.runsInterrupted} / ${m.runsCancelled}`, + `- Failed runs whose task finished by itself / did not / still pending: ${m.runsFailedRecovered} / ${m.runsFailedUnrecovered} / ${m.runsFailedPending}`, `- Running agents peak / mean: ${m.peakRunning} / ${m.meanRunning ?? "not measured"}`, `- Queue wait p50 / p95 / p99: ${dist(m.queueWaitMs, seconds)}`, `- Time to first token p50 / p95 / p99: ${dist(m.ttftMs, seconds)}`, `- First activity p50 / p95 / p99: ${dist(m.firstActivityMs, seconds)}`, `- Run duration p50 / p95 / p99: ${dist(m.runDurationMs, seconds)}`, `- Board API p50 / p95 / p99: ${dist(m.board.ms, millis)}; failed ${m.board.failures} of ${m.board.requests}`, + ...Object.entries(m.board.byRoute).map(([route, d]) => ` - ${route} p50 / p90 / p99: ${millis(d.p50)} / ${millis(d.p90)} / ${millis(d.p99)} (n=${d.n})`), `- Event-loop utilization max: ${m.server.eventLoopUtilizationMax ?? "not measured"}; delay p99 max: ${millis(m.server.eventLoopDelayP99MsMax)}; statements/s max: ${m.server.statementsPerSecondMax ?? "not measured"}; pool wait p99 max: ${millis(m.server.poolWaitP99MsMax)}`, "- Criteria (final):", ...step.criteriaFinal.map(criterionLine), @@ -157,7 +169,9 @@ export function summaryLines(result: RampResult): string[] { if (result.cleanup) { const c = result.cleanup; lines.push(`Cleanup in ${seconds(c.durationMs)}: ${c.runsCancelled} runs cancelled, ${c.tasksArchived} tasks archived, ${c.agentsDeleted} agents deleted` - + ` (${c.agentsKeptForCostHistory} kept terminated for cost history), sandbox leases still active: ${c.activeSandboxLeases ?? "not readable"}, errors: ${c.errors.length}.`); + + ` (${c.agentsKeptForCostHistory} kept terminated for cost history, ${c.agentsStillTerminating.length} still terminating),` + + ` sandbox leases still active: ${c.activeSandboxLeases ?? "not readable"} (${c.sandboxLeasesReleasedDuringWait} released while the cleanup waited), errors: ${c.errors.length}.`); + for (const error of c.errors) lines.push(`- ${error}`); } if (result.error) lines.push(`Error: ${result.error}`); return lines; diff --git a/scripts/scale-ramp/scale-ramp-spend.test.ts b/scripts/scale-ramp/scale-ramp-spend.test.ts new file mode 100644 index 00000000000..73cd1453139 --- /dev/null +++ b/scripts/scale-ramp/scale-ramp-spend.test.ts @@ -0,0 +1,132 @@ +import assert from "node:assert/strict"; +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import { describe, it } from "node:test"; +import { PaperclipClient, Redactor } from "./client.ts"; +import { parseOptions, QUALIFIED_OPENCODE_MODEL } from "./config.ts"; +import { checkSpend, estimateSpend } from "./estimate.ts"; +import { EXIT, main } from "./main.ts"; +import { Ramp } from "./ramp.ts"; +import { prepareCompany } from "./setup.ts"; +import { emptyState, fileStateStore, type RunState } from "./state.ts"; +import { baseArgs, goodMetrics, harness, METRICS_URL, ORIGIN, PROVIDER_KEY, readResult, tempDir, TOKEN } from "./test-harness.ts"; + +describe("dry run and spend", () => { + it("prints the plan and estimate and touches nothing", async () => { + const out = await tempDir(); + const h = harness(); + const code = await main(["--base-url", ORIGIN, "--target-kind", "test-stack", "--out", out, "--dry-run"], h.deps); + assert.equal(code, EXIT.pass); + assert.equal(h.fetchCalls(), 0); + assert.deepEqual(await readdir(out), []); + const text = h.out.join("\n"); + assert.match(text, /Ramp: 10 -> 50 -> 100 -> 200 -> 500 -> 1000 agents/); + assert.match(text, /Estimated spend: \$\d/); + assert.match(text, /retry_needed_zero \[finish-line\]/); + assert.match(text, /Dry run: nothing was created or contacted/); + }); + + it("estimates tasks, sandbox-hours and cost from the steps", () => { + const options = parseOptions(["--base-url", ORIGIN, "--target-kind", "test-stack", "--steps", "10,50", "--hold-minutes", "10", "--task-minutes", "5", + "--drain-minutes", "10"], {}, 0); + const estimate = estimateSpend(options); + assert.equal(estimate.agents, 50); + assert.equal(estimate.tasks, 10 * 2 + 50 * 2); + assert.equal(estimate.maxTasks, 150); + assert.equal(estimate.sandboxHours, 18.3); + assert.equal(estimate.modelUsd, 3); + assert.equal(estimate.totalUsd, 4.83); + }); + + it("requires --confirm-spend of at least the estimate above the threshold", async () => { + const estimate = { totalUsd: 120 } as ReturnType; + assert.equal(checkSpend(estimate, 5, null).ok, false); + assert.equal(checkSpend(estimate, 5, 100).ok, false); + assert.equal(checkSpend(estimate, 5, 130).ok, true); + assert.equal(checkSpend(estimate, 200, null).ok, true); + const out = await tempDir(); + const h = harness(); + const code = await main(["--base-url", ORIGIN, "--target-kind", "test-stack", "--out", out, "--yes"], h.deps); + assert.equal(code, EXIT.usage); + assert.equal(h.fetchCalls(), 0); + assert.match(h.err.join("\n"), /--confirm-spend/); + }); + + it("defaults to the qualified OpenCode model of the runner", async () => { + const source = await readFile(path.join(import.meta.dirname, "../../packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts"), "utf8"); + const match = /QUALIFIED_OPENCODE_MODEL\s*=\s*"([^"]+)"/.exec(source); + assert.equal(match?.[1], QUALIFIED_OPENCODE_MODEL); + }); +}); + +describe("recorded model spend", () => { + it("records model spend from Paperclip's cost records, which the run list does not carry", async () => { + const out = await tempDir(); + const h = harness({ costPerRunUsd: 0.25 }); + await main(baseArgs(out, ["--steps", "2", "--drain-minutes", "0"]), h.deps); + const result = await readResult(out); + const finished = [...h.fake.runs.values()].filter(run => run.status === "succeeded").length; + assert.ok(finished > 0); + assert.equal(result.recordedSpendUsd, Math.round(finished * 25) / 100); + assert.equal(result.steps[0]!.metrics.recordedSpendUsd, result.recordedSpendUsd); + const capped = await tempDir(); + const h2 = harness({ costPerRunUsd: 3 }); + await main(baseArgs(capped, ["--steps", "2", "--hold-minutes", "20"]), h2.deps); + assert.equal((await readResult(capped)).stop!.criterion, "spend_cap"); + }); + + it("keeps the spend of a ramp that crosses a month boundary", async () => { + const out = await tempDir(); + const h = harness({ costPerRunUsd: 0.25 }); + h.clock.t = Date.parse("2026-11-30T23:58:00.000Z"); + await main(baseArgs(out, ["--steps", "2"]), h.deps); + const result = await readResult(out); + const finished = [...h.fake.runs.values()].filter(run => run.status === "succeeded"); + assert.ok(finished.some(run => new Date(run.finishedAt!).getUTCMonth() === 10), "some spend is in the earlier month"); + assert.ok(finished.some(run => new Date(run.finishedAt!).getUTCMonth() === 11), "some spend is in the later month"); + assert.equal(result.recordedSpendUsd, Math.round(finished.length * 25) / 100); + }); + + it("grades a run incomplete when some cost events have no known price", async () => { + const out = await tempDir(); + const h = harness({ metricsUrl: METRICS_URL, metrics: goodMetrics, unpricedEventsPerAgent: 1 }, { PAPERCLIP_SCALE_METRICS_URL: METRICS_URL }); + await main(baseArgs(out, ["--steps", "2"]), h.deps); + const result = await readResult(out); + assert.equal(result.verdict, "incomplete"); + assert.ok(result.incompleteReasons.some(reason => /2 cost events have no known price/.test(reason)), result.incompleteReasons.join("; ")); + assert.match(result.sources.modelSpend!, /no known price/); + assert.equal(result.unpricedCostEvents, 2); + assert.match(await readFile(path.join(out, "scale-test-run", "SUMMARY.md"), "utf8"), /incomplete: 2 cost events have no known price/); + assert.ok(h.out.some(line => /^Warning: \d+ cost events of this run have no known price/.test(line))); + }); + + it("stops on a conservative spend bound when cost events have no known price", async () => { + const out = await tempDir(); + const h = harness({ costPerRunUsd: 0, unpricedEventsPerAgent: 1 }, { PAPERCLIP_SCALE_MODEL_USD_PER_TASK: "0.5" }); + const options = parseOptions(baseArgs(out, ["--steps", "2", "--hold-minutes", "20"]), h.deps.env, h.clock.now()); + const api = new PaperclipClient(options.target, TOKEN, h.deps.fetcher, h.clock, new Redactor([TOKEN])); + const state: RunState = emptyState(options.label, options.target.baseUrl, h.clock.now()); + const store = fileStateStore(options.outDir); + await prepareCompany({ api, options, state, store, credential: PROVIDER_KEY, log: () => {} }); + const outcome = await new Ramp({ api, clock: h.clock, fetcher: h.deps.fetcher, options, state, store, maxTasks: 100, spendCapUsd: 2, metricsToken: null, + log: () => {}, shouldStop: () => false }).run(); + assert.equal(outcome.recordedSpendUsd, 0); + assert.equal(outcome.stop!.criterion, "spend_cap"); + assert.match(outcome.stop!.reason, /no known price.*passed the cap of \$2/); + assert.ok(outcome.tasksCreated >= 5 && outcome.tasksCreated <= 6, `tasks: ${outcome.tasksCreated}`); + }); + + it("stops when recorded spend cannot be read, and reports spend as not measured", async () => { + const out = await tempDir(); + const h = harness({ failRequest: (method, route, count) => method === "GET" && route.endsWith("/costs/by-agent") && count > 2 + ? { status: 503, body: "{\"error\":\"unavailable\"}" } : null }); + const code = await main(baseArgs(out, ["--steps", "2", "--hold-minutes", "20"]), h.deps); + const result = await readResult(out); + assert.equal(code, EXIT.fail); + assert.equal(result.stop!.criterion, "spend_unreadable"); + assert.match(result.stop!.reason, /could not be read 3 times in a row/); + assert.equal(result.recordedSpendUsd, null); + assert.match(result.sources.modelSpend!, /not measured/); + }); + +}); diff --git a/scripts/scale-ramp/scale-ramp.test.ts b/scripts/scale-ramp/scale-ramp.test.ts index 3eab4e9ac87..a4dae632a3f 100644 --- a/scripts/scale-ramp/scale-ramp.test.ts +++ b/scripts/scale-ramp/scale-ramp.test.ts @@ -1,100 +1,21 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; -import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; +import { readdir, readFile } from "node:fs/promises"; import path from "node:path"; -import { after, describe, it } from "node:test"; +import { describe, it } from "node:test"; +import { cleanupRun } from "./cleanup.ts"; import { PaperclipClient, Redactor } from "./client.ts"; import { metricsCredential, parseOptions, QUALIFIED_OPENCODE_MODEL, readSecrets } from "./config.ts"; import { evaluate, type StepMetrics } from "./criteria.ts"; -import { checkSpend, estimateSpend } from "./estimate.ts"; -import { FakePaperclip, VirtualClock, type FakeConfig } from "./fake-paperclip.ts"; import { classifyHost, resolveTarget, TargetRefused } from "./guard.ts"; -import { EXIT, main, type MainDeps } from "./main.ts"; +import { EXIT, main } from "./main.ts"; import { Ramp, serverMetrics } from "./ramp.ts"; -import { verdictOf, type RampResult } from "./report.ts"; +import { verdictOf } from "./report.ts"; import { ensureAgents, prepareCompany } from "./setup.ts"; import { distribution } from "./stats.ts"; -import { emptyState, fileStateStore, mapLimit, type RunState } from "./state.ts"; +import { agentName, emptyState, fileStateStore, mapLimit, type RunState } from "./state.ts"; import { runTiming } from "./workload.ts"; - -const TOKEN = "board-token-SECRET-4f2a9c"; -const PROVIDER_KEY = "sk-or-v1-SECRET-provider-key-77"; -const METRICS_TOKEN = "metrics-token-SECRET-91"; -const HOST = "scale1.test.paperclip.app"; -const ORIGIN = `https://${HOST}`; -const METRICS_URL = "https://metrics.scale1.test/metrics"; - -const dirs: string[] = []; -after(async () => { - for (const dir of dirs) await rm(dir, { recursive: true, force: true }); -}); - -async function tempDir(): Promise { - const dir = await mkdtemp(path.join(tmpdir(), "scale-ramp-test-")); - dirs.push(dir); - return dir; -} - -interface Harness { - clock: VirtualClock; - fake: FakePaperclip; - out: string[]; - err: string[]; - deps: MainDeps; - fetchCalls: () => number; - interrupt: () => void; -} - -function harness(config: Partial = {}, env: Record = {}, answer = HOST): Harness { - const clock = new VirtualClock(); - const fake = new FakePaperclip(clock, { token: TOKEN, origin: ORIGIN, ...config }); - const out: string[] = []; - const err: string[] = []; - let calls = 0; - let handler: (() => void) | null = null; - const deps: MainDeps = { - env: { PAPERCLIP_SCALE_TOKEN: TOKEN, OPENROUTER_API_KEY: PROVIDER_KEY, ...env }, - fetcher: async (url, init) => { - calls += 1; - return fake.fetch(url, init); - }, - clock, - out: line => out.push(line), - err: line => err.push(line), - confirm: async () => answer, - onInterrupt: fn => { - handler = fn; - return () => { - handler = null; - }; - }, - random: () => 0.5, - }; - return { clock, fake, out, err, deps, fetchCalls: () => calls, interrupt: () => handler?.() }; -} - -const baseArgs = (out: string, extra: string[] = []) => [ - "--base-url", ORIGIN, "--target-kind", "test-stack", "--out", out, "--label", "scale-test-run", "--yes", - "--hold-minutes", "2", "--drain-minutes", "3", "--poll-seconds", "15", "--board-probe-seconds", "10", "--task-minutes", "1", ...extra, -]; - -async function readResult(out: string): Promise { - return JSON.parse(await readFile(path.join(out, "scale-test-run", "result.json"), "utf8")) as RampResult; -} - -function assertCleanedUp(fake: FakePaperclip): void { - assert.equal(fake.liveRuns().length, 0, "no queued or running run is left"); - assert.equal(fake.agents.size, 0, "every harness agent is deleted"); - assert.ok([...fake.issues.values()].every(issue => issue.hiddenAt), "every task is archived"); - assert.equal(fake.secrets.size, 0, "the stored provider secret is deleted"); - assert.ok([...fake.projects.values()].every(project => project.archivedAt), "the project is archived"); -} - -const goodMetrics = (running: number) => ({ - eventLoopUtilization: 0.2 + running / 1_000, mainThreadCpu: 0.25, eventLoopDelayP99Ms: 20, statementsPerSecond: 40 * running, - poolWaitP99Ms: 2, runEventLatenessP99Ms: 120, agentApiPreHandlerP99Ms: 15, leaseRenewalsMissed: 0, -}); +import { assertCleanedUp, baseArgs, goodMetrics, harness, HOST, METRICS_TOKEN, METRICS_URL, ORIGIN, PROVIDER_KEY, readResult, tempDir, TOKEN } from "./test-harness.ts"; describe("target guard", () => { it("classifies hosts", () => { @@ -147,61 +68,14 @@ describe("target guard", () => { }); }); -describe("dry run and spend", () => { - it("prints the plan and estimate and touches nothing", async () => { - const out = await tempDir(); - const h = harness(); - const code = await main(["--base-url", ORIGIN, "--target-kind", "test-stack", "--out", out, "--dry-run"], h.deps); - assert.equal(code, EXIT.pass); - assert.equal(h.fetchCalls(), 0); - assert.deepEqual(await readdir(out), []); - const text = h.out.join("\n"); - assert.match(text, /Ramp: 10 -> 50 -> 100 -> 200 -> 500 -> 1000 agents/); - assert.match(text, /Estimated spend: \$\d/); - assert.match(text, /retry_needed_zero \[finish-line\]/); - assert.match(text, /Dry run: nothing was created or contacted/); - }); - - it("estimates tasks, sandbox-hours and cost from the steps", () => { - const options = parseOptions(["--base-url", ORIGIN, "--target-kind", "test-stack", "--steps", "10,50", "--hold-minutes", "10", "--task-minutes", "5", - "--drain-minutes", "10"], {}, 0); - const estimate = estimateSpend(options); - assert.equal(estimate.agents, 50); - assert.equal(estimate.tasks, 10 * 2 + 50 * 2); - assert.equal(estimate.maxTasks, 150); - assert.equal(estimate.sandboxHours, 18.3); - assert.equal(estimate.modelUsd, 3); - assert.equal(estimate.totalUsd, 4.83); - }); - - it("requires --confirm-spend of at least the estimate above the threshold", async () => { - const estimate = { totalUsd: 120 } as ReturnType; - assert.equal(checkSpend(estimate, 5, null).ok, false); - assert.equal(checkSpend(estimate, 5, 100).ok, false); - assert.equal(checkSpend(estimate, 5, 130).ok, true); - assert.equal(checkSpend(estimate, 200, null).ok, true); - const out = await tempDir(); - const h = harness(); - const code = await main(["--base-url", ORIGIN, "--target-kind", "test-stack", "--out", out, "--yes"], h.deps); - assert.equal(code, EXIT.usage); - assert.equal(h.fetchCalls(), 0); - assert.match(h.err.join("\n"), /--confirm-spend/); - }); - - it("defaults to the qualified OpenCode model of the runner", async () => { - const source = await readFile(path.join(import.meta.dirname, "../../packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts"), "utf8"); - const match = /QUALIFIED_OPENCODE_MODEL\s*=\s*"([^"]+)"/.exec(source); - assert.equal(match?.[1], QUALIFIED_OPENCODE_MODEL); - }); -}); - describe("criteria", () => { const empty = distribution([]); const metrics = (patch: Partial = {}): StepMetrics => ({ step: { index: 0, concurrency: 10, holdMs: 60_000 }, startedAt: 0, endedAt: 1, tasksCreated: 10, tasksStarted: 10, tasksFinished: 10, - tasksFailed: 0, tasksNeedingRetry: 0, runsFailed: 0, runsInterrupted: 0, runsCancelled: 0, peakRunning: 10, meanRunning: 9, + tasksFailed: 0, tasksNeedingRetry: 0, runsEnded: 10, runsFailed: 0, runsFailedRecovered: 0, runsFailedUnrecovered: 0, runsFailedPending: 0, + runsInterrupted: 0, runsCancelled: 0, peakRunning: 10, meanRunning: 9, queueWaitMs: distribution([1_000, 2_000]), taskToStartMs: empty, ttftMs: empty, firstActivityMs: distribution([4_000]), runDurationMs: empty, - board: { requests: 10, failures: 0, ms: distribution([100, 120, 130, 140]) }, + board: { requests: 10, failures: 0, ms: distribution([100, 120, 130, 140]), byRoute: {} }, server: { mainThreadCpuMax: null, eventLoopUtilizationMax: null, eventLoopDelayP99MsMax: null, statementsPerSecondMax: null, poolWaitP99MsMax: null, runEventLatenessP99MsMax: null, agentApiPreHandlerP99MsMax: null, leaseRenewalsMissed: null, samples: 0 }, recordedSpendUsd: 0, ...patch, @@ -217,22 +91,41 @@ describe("criteria", () => { assert.equal(results.lease_renewals_missed_zero, "not_measured"); assert.equal(results.run_event_lateness_p99, "not_measured"); assert.equal(results.kill_one_process_resumes, "not_measured"); - assert.equal(byId(evaluate(metrics({ board: { requests: 0, failures: 0, ms: empty } }), thresholds)).board_api_p90, "not_measured"); + assert.equal(byId(evaluate(metrics({ board: { requests: 0, failures: 0, ms: empty, byRoute: {} } }), thresholds)).board_api_p90, "not_measured"); }); it("fails each finish-line threshold it can measure", () => { const server = { mainThreadCpuMax: 0.7, eventLoopUtilizationMax: 0.5, eventLoopDelayP99MsMax: 10, statementsPerSecondMax: 1, poolWaitP99MsMax: 1, runEventLatenessP99MsMax: 1_500, agentApiPreHandlerP99MsMax: 100, leaseRenewalsMissed: 2, samples: 3 }; - const results = byId(evaluate(metrics({ tasksNeedingRetry: 1, peakRunning: 8, board: { requests: 4, failures: 1, ms: distribution([100, 600, 700]) }, server }), thresholds)); + const results = byId(evaluate(metrics({ tasksNeedingRetry: 1, peakRunning: 8, runsFailed: 1, runsFailedUnrecovered: 1, + board: { requests: 4, failures: 1, ms: distribution([100, 600, 700]), byRoute: {} }, server }), thresholds)); assert.deepEqual(results, { - retry_needed_zero: "fail", failed_runs_zero: "pass", tasks_rested_not_done_zero: "pass", concurrency_reached: "fail", board_api_p90: "fail", board_api_errors_zero: "fail", + retry_needed_zero: "fail", failed_runs_unrecovered_zero: "fail", failed_run_rate: "fail", tasks_rested_not_done_zero: "pass", concurrency_reached: "fail", + board_api_p90: "fail", board_api_errors_zero: "fail", queue_wait_p95: "pass", first_activity_p95: "pass", main_thread_cpu: "fail", run_event_lateness_p99: "fail", agent_api_pre_handler_p99: "fail", lease_renewals_missed_zero: "fail", kill_one_process_resumes: "not_measured", }); }); - it("only evaluates retry and failed runs as immediate stops", () => { - assert.deepEqual(evaluate(metrics(), thresholds, true).map(result => result.id), ["retry_needed_zero", "failed_runs_zero"]); + it("only evaluates retry and unrecovered failed runs as immediate stops", () => { + assert.deepEqual(evaluate(metrics(), thresholds, true).map(result => result.id), ["retry_needed_zero", "failed_runs_unrecovered_zero"]); + }); + + it("passes a step whose failed runs all recovered while their share of ended runs stays under the limit", () => { + const recovered = byId(evaluate(metrics({ runsEnded: 106, runsFailed: 1, runsFailedRecovered: 1 }), thresholds)); + assert.equal(recovered.failed_runs_unrecovered_zero, "pass"); + assert.equal(recovered.failed_run_rate, "pass"); + assert.equal(byId(evaluate(metrics({ runsEnded: 20, runsFailed: 1, runsFailedRecovered: 1 }), thresholds)).failed_run_rate, "fail"); + assert.equal(byId(evaluate(metrics({ runsEnded: 0 }), thresholds)).failed_run_rate, "not_measured"); + }); + + it("compares the unrounded failed-run share with the limit", () => { + const justOver = evaluate(metrics({ runsEnded: 49, runsFailed: 1, runsFailedRecovered: 1 }), thresholds).find(result => result.id === "failed_run_rate")!; + assert.equal(justOver.status, "fail"); + assert.equal(justOver.observed, 2); + const zero = parseOptions(["--base-url", ORIGIN, "--target-kind", "test-stack", "--max-failed-run-percent", "0"], {}, 0).thresholds; + assert.equal(byId(evaluate(metrics({ runsEnded: 2_500, runsFailed: 1, runsFailedRecovered: 1 }), zero)).failed_run_rate, "fail"); + assert.equal(byId(evaluate(metrics({ runsEnded: 50, runsFailed: 1, runsFailedRecovered: 1 }), thresholds)).failed_run_rate, "pass"); }); it("reads first token and first activity from run spans, with an event fallback", () => { @@ -404,7 +297,7 @@ describe("ramp against the fake API", () => { mutate: (issue, run, now) => { issue.status = "cancelled"; run.status = "cancelled"; run.finishedAt = now; } }, { name: "blocked", criterion: "run_started", reason: /ended before any run started.*is blocked.*is queued/, mutate: issue => { issue.status = "blocked"; } }, - { name: "failed before it started", criterion: "failed_runs_zero", reason: /failed_runs_zero failed/, + { name: "failed before it started", criterion: "run_started", reason: /ended before any run started.*is failed/, mutate: (_issue, run, now) => { run.status = "failed"; run.finishedAt = now; } }, ]; for (const preflight of preflightCases) { @@ -450,11 +343,11 @@ describe("ramp against the fake API", () => { const out = await tempDir(); const h = harness({ failWhenRunningAbove: 0, runDurationMs: 3 * 60_000, agentReadyDelayMs: index => index === 0 ? 3_000 : 10 * 60_000 }); const started = h.clock.now(); - const code = await main(baseArgs(out, ["--steps", "1,3", "--min-concurrency-ratio", "0.5"]), h.deps); + const code = await main(baseArgs(out, ["--steps", "1,3", "--min-concurrency-ratio", "0.5", "--recovery-grace-minutes", "1"]), h.deps); const result = await readResult(out); assert.equal(code, EXIT.fail); assert.equal(result.stop!.stepIndex, 1); - assert.equal(result.stop!.criterion, "failed_runs_zero"); + assert.equal(result.stop!.criterion, "failed_runs_unrecovered_zero"); assert.equal(result.steps[0]!.metrics.runsFailed, 1); assert.ok(h.clock.now() - started < 6 * 60_000, "the failure is seen during the agent wait, not after it"); assertCleanedUp(h.fake); @@ -540,7 +433,7 @@ describe("ramp against the fake API", () => { assert.equal(code, EXIT.fail); assert.equal(result.verdict, "fail"); assert.equal(result.stop!.stepIndex, 1); - assert.equal(result.stop!.criterion, "failed_runs_zero"); + assert.equal(result.stop!.criterion, "failed_run_rate"); assert.equal(result.steps.length, 2); const agents = h.fake.requests.filter(request => request.method === "POST" && new URL(request.url).pathname.endsWith("/agents")); assert.equal(agents.length, 4, "the 8-agent step never created agents"); @@ -554,7 +447,7 @@ describe("ramp against the fake API", () => { const result = await readResult(out); assert.equal(result.verdict, "fail"); assert.equal(result.stop!.stepIndex, 0); - assert.ok(["retry_needed_zero", "failed_runs_zero"].includes(result.stop!.criterion!)); + assert.ok(["retry_needed_zero", "failed_runs_unrecovered_zero"].includes(result.stop!.criterion!)); assert.ok(result.steps[0]!.metrics.tasksNeedingRetry + result.steps[0]!.metrics.runsFailed > 0); }); @@ -625,6 +518,138 @@ describe("ramp against the fake API", () => { assert.ok(result.drained, "finished runs are not left counted as running"); }); + it("keeps ramping when Paperclip recovers a failed run by itself, counts it, and grades the failed-run rate at the step end", async () => { + const out = await tempDir(); + const h = harness({ failRun: index => index === 2, recoverFailedRunAfterMs: 30_000, runDurationMs: 20_000 }); + await main(baseArgs(out, ["--steps", "2", "--hold-minutes", "4"]), h.deps); + const result = await readResult(out); + const metrics = result.steps[0]!.metrics; + assert.equal(metrics.runsFailed, 1); + assert.equal(metrics.runsFailedRecovered, 1); + assert.equal(metrics.runsFailedUnrecovered, 0); + assert.equal(metrics.tasksNeedingRetry, 0, "a scheduled automatic retry is not a task that needs Retry"); + assert.equal(result.stop!.criterion, "failed_run_rate"); + assert.match(result.stop!.reason, /at the end of the step/); + assert.ok(metrics.tasksFinished >= 3, "the step kept creating work after the failure"); + const tolerant = await tempDir(); + const h2 = harness({ failRun: index => index === 2, recoverFailedRunAfterMs: 30_000, runDurationMs: 20_000 }); + await main(baseArgs(tolerant, ["--steps", "2", "--hold-minutes", "4", "--max-failed-run-percent", "50", "--min-concurrency-ratio", "0.5"]), h2.deps); + const passed = await readResult(tolerant); + assert.equal(passed.stop, null); + assert.ok(passed.steps[0]!.criteriaFinal.filter(c => c.id.startsWith("failed_run")).every(c => c.status === "pass")); + }); + + it("stops when a failed run's task has not finished within the recovery grace", async () => { + const out = await tempDir(); + const h = harness({ failRun: index => index === 2 }); + const started = h.clock.now(); + const code = await main(baseArgs(out, ["--steps", "2", "--hold-minutes", "20", "--recovery-grace-minutes", "2"]), h.deps); + const result = await readResult(out); + assert.equal(code, EXIT.fail); + assert.equal(result.stop!.criterion, "failed_runs_unrecovered_zero"); + assert.equal(result.steps[0]!.metrics.runsFailedUnrecovered, 1); + assert.ok(h.clock.now() - started < 10 * 60_000, "it stops once the grace runs out, not at the end of the hold"); + }); + + it("counts a run that Paperclip retries on the same run id once, as recovered, with a share of at most 100%", async () => { + const out = await tempDir(); + const h = harness({ failRun: index => index === 2, retrySameRunAfterMs: 30_000, runDurationMs: 20_000 }); + await main(baseArgs(out, ["--steps", "2", "--hold-minutes", "4", "--max-failed-run-percent", "50", "--min-concurrency-ratio", "0.5"]), h.deps); + const result = await readResult(out); + const metrics = result.steps[0]!.metrics; + const retried = [...h.fake.runs.values()].find(run => run.recovery)!; + assert.equal(retried.status, "succeeded", "the failed run itself finished on its retry"); + assert.equal(metrics.runsFailed, 1); + assert.equal(metrics.runsFailedRecovered, 1); + assert.equal(metrics.tasksNeedingRetry, 0); + assert.ok(metrics.runsEnded >= metrics.runsFailed); + const rate = result.steps[0]!.criteriaFinal.find(criterion => criterion.id === "failed_run_rate")!; + assert.equal(rate.observed, Math.round(1_000 / metrics.runsEnded) / 10); + assert.equal(rate.status, "pass"); + }); + + it("keeps a failed run in the ended runs while Paperclip retries it on the same run id", async () => { + const out = await tempDir(); + const h = harness({ failRun: index => index === 2, retrySameRunAfterMs: 30_000, sameRunRetryDurationMs: 60 * 60_000, runDurationMs: 20_000 }); + await main(baseArgs(out, ["--steps", "2", "--hold-minutes", "4", "--drain-minutes", "0", "--max-failed-run-percent", "50", + "--min-concurrency-ratio", "0.5"]), h.deps); + const metrics = (await readResult(out)).steps[0]!.metrics; + const succeeded = [...h.fake.runs.values()].filter(run => run.status === "succeeded").length; + assert.equal(metrics.runsFailed, 1); + assert.equal(metrics.runsFailedPending, 1, "the retry is still running"); + assert.equal(metrics.runsEnded, succeeded + 1, "the retrying run still counts as ended once"); + }); + + it("keeps the termination wait to a bounded number of reads for a large run", async () => { + const out = await tempDir(); + const h = harness({ terminationDelayMs: 60 * 60_000 }); + const options = parseOptions(baseArgs(out, ["--steps", "2"]), h.deps.env, h.clock.now()); + const api = new PaperclipClient(options.target, TOKEN, h.deps.fetcher, h.clock, new Redactor([TOKEN])); + const state: RunState = emptyState(options.label, options.target.baseUrl, h.clock.now()); + await prepareCompany({ api, options, state, store: fileStateStore(options.outDir), credential: PROVIDER_KEY, log: () => {} }); + for (let index = 0; index < 1_000; index += 1) { + const agent = await api.post<{ id: string }>(`/api/companies/${state.companyId}/agents`, { name: agentName(options.label, index) }); + state.agentIds.push(agent.id); + } + const before = h.fake.requests.length; + const report = await cleanupRun({ api, clock: h.clock, label: options.label, state }); + const reads = h.fake.requests.slice(before).filter(request => request.method === "GET" && /\/api\/agents\/[0-9a-f-]+$/.test(new URL(request.url).pathname)).length; + assert.equal(report.agentsStillTerminating.length, 1_000); + assert.ok(reads <= 5_000, `termination reads: ${reads}`); + }); + + it("reports board latency per route", async () => { + const out = await tempDir(); + const h = harness({ boardLatencyMs: () => 300 }); + await main(baseArgs(out, ["--steps", "2"]), h.deps); + const byRoute = (await readResult(out)).steps[0]!.metrics.board.byRoute; + assert.deepEqual(Object.keys(byRoute).sort(), ["agents", "dashboard", "issues", "live-runs", "sidebar-badges"]); + assert.equal(byRoute.dashboard!.p50, 300); + assert.ok(byRoute.issues!.n > 0); + const summary = await readFile(path.join(out, "scale-test-run", "SUMMARY.md"), "utf8"); + assert.match(summary, / - dashboard p50 \/ p90 \/ p99: 300 ms/); + }); + + it("deletes an agent only after Paperclip completes its termination", async () => { + const out = await tempDir(); + const h = harness({ terminationDelayMs: 2 * 60_000 }); + await main(baseArgs(out, ["--steps", "2"]), h.deps); + const result = await readResult(out); + assert.deepEqual(result.cleanup!.errors, []); + assert.deepEqual(result.cleanup!.agentsStillTerminating, []); + assert.equal(result.cleanup!.agentsDeleted, 2); + assertCleanedUp(h.fake); + }); + + it("names the agents still terminating and the sandbox leases still active, and finishes them with --cleanup-only later", async () => { + const out = await tempDir(); + const h = harness({ terminationDelayMs: 20 * 60_000, leaseLingerMs: 20 * 60_000 }); + const code = await main(baseArgs(out, ["--steps", "2"]), h.deps); + const result = await readResult(out); + assert.equal(code, EXIT.incomplete); + assert.equal(result.cleanup!.agentsStillTerminating.length, 2); + assert.equal(result.cleanup!.agentsDeleted, 0); + assert.ok(result.cleanup!.activeSandboxLeases! > 0); + assert.equal(result.cleanup!.activeSandboxLeaseIds.length, result.cleanup!.activeSandboxLeases); + assert.ok(result.cleanup!.errors.some(error => /had not completed termination/.test(error))); + assert.ok(result.cleanup!.errors.some(error => /still active after 5 min: lease-/.test(error))); + assert.ok(!result.cleanup!.errors.some(error => /409/.test(error)), "no delete is sent before termination completes"); + await h.clock.sleep(20 * 60_000); + const later = await main(["--base-url", ORIGIN, "--target-kind", "test-stack", "--out", out, "--cleanup-only", "scale-test-run", "--yes"], h.deps); + assert.equal(later, EXIT.pass); + assertCleanedUp(h.fake); + }); + + it("counts sandbox leases that release while the cleanup waits", async () => { + const out = await tempDir(); + const h = harness({ leaseLingerMs: 2 * 60_000 }); + await main(baseArgs(out, ["--steps", "2"]), h.deps); + const cleanup = (await readResult(out)).cleanup!; + assert.equal(cleanup.activeSandboxLeases, 0); + assert.ok(cleanup.sandboxLeasesReleasedDuringWait > 0); + assert.deepEqual(cleanup.errors, []); + }); + it("cleans up after a crash in the middle of the ramp", async () => { const out = await tempDir(); const h = harness({ failRequest: (method, route, count) => method === "POST" && route.endsWith("/issues") && count === 5 ? { status: 500, body: "boom" } : null }); diff --git a/scripts/scale-ramp/test-harness.ts b/scripts/scale-ramp/test-harness.ts new file mode 100644 index 00000000000..26229693107 --- /dev/null +++ b/scripts/scale-ramp/test-harness.ts @@ -0,0 +1,86 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { after } from "node:test"; +import { FakePaperclip, VirtualClock, type FakeConfig } from "./fake-paperclip.ts"; +import type { MainDeps } from "./main.ts"; +import type { RampResult } from "./report.ts"; + +export const TOKEN = "board-token-SECRET-4f2a9c"; +export const PROVIDER_KEY = "sk-or-v1-SECRET-provider-key-77"; +export const METRICS_TOKEN = "metrics-token-SECRET-91"; +export const HOST = "scale1.test.paperclip.app"; +export const ORIGIN = `https://${HOST}`; +export const METRICS_URL = "https://metrics.scale1.test/metrics"; + +const dirs: string[] = []; +after(async () => { + for (const dir of dirs) await rm(dir, { recursive: true, force: true }); +}); + +export async function tempDir(): Promise { + const dir = await mkdtemp(path.join(tmpdir(), "scale-ramp-test-")); + dirs.push(dir); + return dir; +} + +export interface Harness { + clock: VirtualClock; + fake: FakePaperclip; + out: string[]; + err: string[]; + deps: MainDeps; + fetchCalls: () => number; + interrupt: () => void; +} + +export function harness(config: Partial = {}, env: Record = {}, answer = HOST): Harness { + const clock = new VirtualClock(); + const fake = new FakePaperclip(clock, { token: TOKEN, origin: ORIGIN, ...config }); + const out: string[] = []; + const err: string[] = []; + let calls = 0; + let handler: (() => void) | null = null; + const deps: MainDeps = { + env: { PAPERCLIP_SCALE_TOKEN: TOKEN, OPENROUTER_API_KEY: PROVIDER_KEY, ...env }, + fetcher: async (url, init) => { + calls += 1; + return fake.fetch(url, init); + }, + clock, + out: line => out.push(line), + err: line => err.push(line), + confirm: async () => answer, + onInterrupt: fn => { + handler = fn; + return () => { + handler = null; + }; + }, + random: () => 0.5, + }; + return { clock, fake, out, err, deps, fetchCalls: () => calls, interrupt: () => handler?.() }; +} + +export const baseArgs = (out: string, extra: string[] = []) => [ + "--base-url", ORIGIN, "--target-kind", "test-stack", "--out", out, "--label", "scale-test-run", "--yes", + "--hold-minutes", "2", "--drain-minutes", "3", "--poll-seconds", "15", "--board-probe-seconds", "10", "--task-minutes", "1", ...extra, +]; + +export async function readResult(out: string): Promise { + return JSON.parse(await readFile(path.join(out, "scale-test-run", "result.json"), "utf8")) as RampResult; +} + +export function assertCleanedUp(fake: FakePaperclip): void { + assert.equal(fake.liveRuns().length, 0, "no queued or running run is left"); + assert.equal(fake.agents.size, 0, "every harness agent is deleted"); + assert.ok([...fake.issues.values()].every(issue => issue.hiddenAt), "every task is archived"); + assert.equal(fake.secrets.size, 0, "the stored provider secret is deleted"); + assert.ok([...fake.projects.values()].every(project => project.archivedAt), "the project is archived"); +} + +export const goodMetrics = (running: number) => ({ + eventLoopUtilization: 0.2 + running / 1_000, mainThreadCpu: 0.25, eventLoopDelayP99Ms: 20, statementsPerSecond: 40 * running, + poolWaitP99Ms: 2, runEventLatenessP99Ms: 120, agentApiPreHandlerP99Ms: 15, leaseRenewalsMissed: 0, +}); diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index 11a49de8470..980f64e64ff 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -1,6 +1,6 @@ { "inputs": { - "package.json": "2d2a4abebb6a5931be1a8f0c8fb2615890324d21", + "package.json": "b6a185b015b2ec9ffc80564f0970916b65b7d6ba", "packages/plugins/sdk/package.json": "b7a4936e2803fd0d3b32dd7f17fc1ea86609ead0", "packages/plugins/sdk/src/bundlers.ts": "883bf3d9f2bd42877cfc0c3707cd882a6f6248d0", "packages/plugins/sdk/src/define-plugin.ts": "a99e07ce77ef72fcf1df1347346aacb8ba12e284",