From bf67e2fbd909fc62e9efbfe70147fd83935d3e6f Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:06:41 -0600 Subject: [PATCH] test(chaos): honor 429 retry delays on harness reads (#20194) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The restart chaos harness collects each wave's speed from the board API after the tasks rest, including the M1 subtasks' runs > - The server rate-limits concurrent issue-list requests per client and answers 429 with `retryAfterSeconds` > - After a 95-task storm, the harness requested every M1 parent's subtask list at once. The server returned 429, speed collection failed, and the wave exited 1 with no speed grade > - This pull request makes harness GET requests honor a 429's retry delay with a bounded number of attempts, and collects subtask speed one parent at a time > - The benefit is that a busy wave keeps its speed grade, and the harness behaves like a polite API client ## Linked Issues or Issue Description Refs #20071, #20147 **What happened?** Wave `w-20261109T055902-s9402` (K13 storm, A1/C1/M1, 95 tasks) finished its tasks. Speed collection then failed with: `GET /api/companies/…/issues?view=index&parentIds=…&limit=20 returned 429: {"error":"Too many concurrent issue-list requests for this actor/client","retryAfterSeconds":1}`. `main.ts` ran `childSpeedTasks()` for every observer in parallel. **Expected behavior** - The harness waits and retries a rate-limited read. - It does not flood the issue-list endpoint while collecting speed. - A busy wave still gets graded. ## What Changed - `api.ts`: `ChaosApi` retries a GET that returns 429. - It waits for the delay from `Retry-After` or the body's `retryAfterSeconds` (default 1 s, capped at 10 s), scaled by the attempt number. - It makes at most 5 attempts by default. - Writes (POST/PUT/PATCH/DELETE) are never retried, so a user action is never repeated. - `rateLimitDelayMs` is exported for tests. - `main.ts`: subtask speed is collected one observer at a time instead of with `Promise.all`. - New `api.test.ts` against a real local HTTP server: - a GET that gets 429 twice, then 200, succeeds; - a POST that gets 429 is sent once; - retries stop at the bound; - delay parsing (header, body, cap, default). ## Verification - `npx vitest run` in `tests/restart-chaos`: 216 passed. - `node scripts/typecheck-test-projects.mjs tests/restart-chaos`: clean. ## Risks - A persistently rate-limited read now takes up to about 15 s before it fails, instead of failing at once. - Collecting subtask speed one parent at a time adds a few seconds to the end of large M1 waves. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- tests/restart-chaos/api.test.ts | 58 +++++++++++++++++++++++++++++++++ tests/restart-chaos/api.ts | 30 +++++++++++++++-- tests/restart-chaos/main.ts | 3 +- 3 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 tests/restart-chaos/api.test.ts diff --git a/tests/restart-chaos/api.test.ts b/tests/restart-chaos/api.test.ts new file mode 100644 index 00000000000..35854cbea8a --- /dev/null +++ b/tests/restart-chaos/api.test.ts @@ -0,0 +1,58 @@ +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { afterEach, describe, expect, it } from "vitest"; +import { ChaosApi, ChaosApiError, rateLimitDelayMs } from "./api.js"; + +let server: Server | null = null; +afterEach(async () => { + await new Promise(resolve => server ? server.close(() => resolve()) : resolve()); + server = null; +}); + +async function serve(handler: (count: number) => { status: number; body: unknown }) { + let count = 0; + const arrivals: number[] = []; + server = createServer((_request, response) => { + count += 1; + arrivals.push(performance.now()); + const reply = handler(count); + response.writeHead(reply.status, { "content-type": "application/json" }); + response.end(JSON.stringify(reply.body)); + }); + await new Promise(resolve => server!.listen(0, "127.0.0.1", () => resolve())); + return { url: `http://127.0.0.1:${(server!.address() as AddressInfo).port}`, count: () => count, arrivals }; +} + +describe("ChaosApi rate limits", () => { + it("retries a GET the server rate-limited, after the delay it asked for", async () => { + const target = await serve(count => count <= 2 + ? { status: 429, body: { error: "Too many concurrent issue-list requests", retryAfterSeconds: 0.05 } } + : { status: 200, body: [{ id: "c1" }] }); + await expect(new ChaosApi(target.url).get("/api/issues")).resolves.toEqual([{ id: "c1" }]); + expect(target.count()).toBe(3); + const [first, second, third] = target.arrivals; + expect(second! - first!).toBeGreaterThanOrEqual(45); + expect(third! - second!).toBeGreaterThanOrEqual(95); + }); + + it("does not repeat a write the server rate-limited", async () => { + const target = await serve(() => ({ status: 429, body: { retryAfterSeconds: 0.01 } })); + await expect(new ChaosApi(target.url).post("/api/issues/i1/comments", { body: "x" })).rejects.toBeInstanceOf(ChaosApiError); + expect(target.count()).toBe(1); + }); + + it("gives up after a bounded number of attempts", async () => { + const target = await serve(() => ({ status: 429, body: { retryAfterSeconds: 0.01 } })); + await expect(new ChaosApi(target.url, 10_000, 3).get("/api/issues")).rejects.toThrow(/returned 429/); + expect(target.count()).toBe(3); + }); + + it("reads the delay from Retry-After or the body, capped, and defaults to one second", () => { + const response = (status: number, retryAfter: string | null = null) => ({ status, headers: { get: () => retryAfter } }); + expect(rateLimitDelayMs(response(200), "")).toBeNull(); + expect(rateLimitDelayMs(response(429, "2"), "")).toBe(2_000); + expect(rateLimitDelayMs(response(429), JSON.stringify({ retryAfterSeconds: 3 }))).toBe(3_000); + expect(rateLimitDelayMs(response(429, "600"), "")).toBe(10_000); + expect(rateLimitDelayMs(response(429), "not json")).toBe(1_000); + }); +}); diff --git a/tests/restart-chaos/api.ts b/tests/restart-chaos/api.ts index 249e59cef15..b5e2fb8619d 100644 --- a/tests/restart-chaos/api.ts +++ b/tests/restart-chaos/api.ts @@ -8,10 +8,29 @@ export class ChaosApiError extends Error { constructor(readonly status: number | null, message: string) { super(message); this.name = "ChaosApiError"; } } +export function rateLimitDelayMs(response: { status: number; headers: { get(name: string): string | null } }, text: string): number | null { + if (response.status !== 429) return null; + let seconds = Number(response.headers.get("retry-after")); + if (!Number.isFinite(seconds) || seconds <= 0) { + try { seconds = Number((JSON.parse(text) as { retryAfterSeconds?: unknown }).retryAfterSeconds); } catch { seconds = NaN; } + } + return Number.isFinite(seconds) && seconds > 0 ? Math.min(10_000, seconds * 1_000) : 1_000; +} + export class ChaosApi { - constructor(readonly baseUrl: string, private readonly timeoutMs = 10_000) {} + constructor(readonly baseUrl: string, private readonly timeoutMs = 10_000, private readonly rateLimitAttempts = 5) {} private async request(method: string, route: string, body?: unknown, timeoutMs = this.timeoutMs): Promise { + for (let attempt = 1; ; attempt += 1) { + const outcome = await this.attempt(method, route, body, timeoutMs); + if (!("retryAfterMs" in outcome)) return outcome.value; + if (method !== "GET" || attempt >= this.rateLimitAttempts) throw outcome.error; + await new Promise(resolve => setTimeout(resolve, Math.min(10_000, outcome.retryAfterMs * attempt))); + } + } + + private async attempt(method: string, route: string, body: unknown, timeoutMs: number): + Promise<{ value: T } | { retryAfterMs: number; error: ChaosApiError }> { let response: Response; try { response = await fetch(`${this.baseUrl}${route}`, { @@ -25,8 +44,13 @@ export class ChaosApi { throw new ChaosApiError(null, `${method} ${route} unreachable: ${error instanceof Error ? error.message : String(error)}`); } const text = await response.text(); - if (!response.ok) throw new ChaosApiError(response.status, `${method} ${route} returned ${response.status}: ${text.slice(0, 500)}`); - return (text ? JSON.parse(text) : null) as T; + if (!response.ok) { + const error = new ChaosApiError(response.status, `${method} ${route} returned ${response.status}: ${text.slice(0, 500)}`); + const retryAfterMs = rateLimitDelayMs(response, text); + if (retryAfterMs !== null) return { retryAfterMs, error }; + throw error; + } + return { value: (text ? JSON.parse(text) : null) as T }; } get(route: string, timeoutMs?: number) { return this.request("GET", route, undefined, timeoutMs); } diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index 2f584a3e888..7c3ac613c10 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -469,7 +469,8 @@ async function main(): Promise { } await writeFile(path.join(waveDir, "faults.json"), `${JSON.stringify(faults, null, 2)}\n`, { mode: 0o600 }); const speed = await (async () => { - const childTasks = (await Promise.all(launched.map(observer => observer.childSpeedTasks()))).flat(); + const childTasks: Awaited> = []; + for (const observer of launched) childTasks.push(...await observer.childSpeedTasks()); return collectWaveSpeed(api, plan.options.env, [...launched.flatMap(observer => observer.record.issueId ? [{ issueId: observer.record.issueId, adapter: observer.plan.adapter, samples: observer.record.samples }] : []), ...childTasks], faults); })().catch(error => { timeline.add("wave_speed_failed", { error: String(error) }); anyFailure = true; return null; });