diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index a58e7afd613..e09f3fcc0e3 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -727,8 +727,9 @@ run's agents, the task view from `/api/issues/:id`, pending questions and approv approvals, failure chains and next events from `/api/heartbeat-runs/:id/events`, and duplicated comments and subtasks from comments and the issue list. Per release the table counts failed runs and the time from the old build's last answer to each surviving run's next event. Web and API continuity are reported for both the health -probe and every API request the harness made: requests, failures (any answer that is not 2xx, or no answer), slow -answers and the longest gap between successful answers. Runs come from the company run list, or from every task's +probe and every API request the harness made: requests, failures, slow answers and the longest gap between +successful answers. A health probe fails unless `/api/health` answers 200 with a readable body. An API request fails +on no answer, a server error, or 401, 403, 408 or 429; a deliberate 400, 404, 409 or 422 answer is not a failure. Runs come from the company run list, or from every task's run list when that list is full, so no task's early runs are lost. A queued or running run with no activity for 5 minutes is not a live owner, the same window the drain uses. diff --git a/tests/restart-chaos/hosted-api.ts b/tests/restart-chaos/hosted-api.ts index c197a1ad35f..47437b4bfe2 100644 --- a/tests/restart-chaos/hosted-api.ts +++ b/tests/restart-chaos/hosted-api.ts @@ -8,6 +8,7 @@ import type { HealthSample } from "./hosted-release.js"; export const HOSTED_TOKEN_ENV = "PAPERCLIP_HOSTED_TOKEN"; const TRANSIENT_STATUSES = new Set([502, 503, 504]); +export const ANSWERED_REFUSALS: ReadonlySet = new Set([400, 404, 409, 422]); export function hostedAuthPaths(env: NodeJS.ProcessEnv = process.env): string[] { const paths = [path.join(os.homedir(), ".paperclip", "auth.json")]; @@ -48,7 +49,7 @@ export class HostedApi { ...(body === undefined ? {} : { "content-type": "application/json" }) }, body: body === undefined ? undefined : JSON.stringify(body) }); const text = await response.text(); - this.requests.push({ at: started, ok: response.ok, reachable: response.status < 500, status: response.status, ms: Date.now() - started, method }); + this.requests.push({ at: started, ok: response.ok || ANSWERED_REFUSALS.has(response.status), reachable: response.status < 500, status: response.status, ms: Date.now() - started, method }); return { status: response.ok ? 200 : response.status, text, retryAfterMs: rateLimitDelayMs(response, text) }; } catch (error) { this.requests.push({ at: started, ok: false, reachable: false, status: null, ms: Date.now() - started, method }); diff --git a/tests/restart-chaos/hosted-main.test.ts b/tests/restart-chaos/hosted-main.test.ts index 0d283fe0937..1322dfd8aa7 100644 --- a/tests/restart-chaos/hosted-main.test.ts +++ b/tests/restart-chaos/hosted-main.test.ts @@ -59,11 +59,12 @@ describe("hosted run control", () => { const dir = await tempDir(); const writes: string[] = []; let agentStatus = "idle"; + let lifecycleReads = 0; const api = { get: async (route: string) => { if (route.endsWith("/environments")) return [{ id: "e1", driver: "sandbox" }] as T; if (route.endsWith("/agents")) return [{ id: "a1", name: "RC rc-x opencode" }] as T; - if (route === "/api/agents/a1") return { id: "a1", status: agentStatus } as T; + if (route === "/api/agents/a1") return { id: "a1", status: agentStatus, lifecycleState: agentStatus === "terminated" && lifecycleReads++ > 0 ? "terminated" : "terminating" } as T; if (route.endsWith("/secrets")) return [{ id: "s1", name: "restart-chaos rc-x OPENROUTER_API_KEY" }] as T; if (route.endsWith("/projects")) return [{ id: "p1", name: "Restart chaos rc-x" }] as T; return [] as T; @@ -85,7 +86,8 @@ describe("hosted run control", () => { await expect(setupHosted({ api, options, companyId: "c1", dir, env: { OPENROUTER_API_KEY: "k" }, record: () => {} })).rejects.toThrow(/agent creation failed/); expect(JSON.parse(await readFile(statePath(dir), "utf8"))).toMatchObject({ companyId: "c1", projectId: "p1", secrets: [{ id: "s1" }] }); writes.length = 0; - expect(await finalizeHosted({ api, keep: false, label: "rc-x", dir, state: null, record: () => {} })).toMatchObject({ errors: [] }); + expect(await finalizeHosted({ api, keep: false, label: "rc-x", dir, state: null, record: () => {}, terminatePollMs: 1 })).toMatchObject({ errors: [] }); + expect(lifecycleReads).toBeGreaterThan(1); expect(writes).toEqual(["PATCH /api/agents/a1", "POST /api/agents/a1/terminate", "DELETE /api/agents/a1", "DELETE /api/secrets/s1", "PATCH /api/projects/p1"]); expect(await finalizeHosted({ api, keep: true, label: "rc-x", dir, state: null, record: () => {} })).toBeNull(); }); @@ -151,11 +153,13 @@ describe("hosted readers collect complete history", () => { }); it("counts a refused request as a failure in continuity, not only server errors", async () => { - const statuses = [200, 401, 429, 503]; + const statuses = [200, 401, 404, 429, 503]; const api = new HostedApi(hostedTarget(`https://${host}`, host), "t", 1_000, (async () => new Response("{}", { status: statuses.shift()! })) as unknown as typeof fetch); await api.get("/api/health"); await api.get("/api/health").catch(() => undefined); expect(continuity(requestSamples(api.requests)).failures).toBeGreaterThanOrEqual(1); expect(api.requests[1]).toMatchObject({ ok: false, reachable: true, status: 401 }); + await api.get("/api/companies/c1/assigned-work-invariant").catch(() => undefined); + expect(api.requests[2]).toMatchObject({ ok: true, status: 404 }); }); }); diff --git a/tests/restart-chaos/hosted-main.ts b/tests/restart-chaos/hosted-main.ts index 1df1e1b5fa4..58d8d01361d 100644 --- a/tests/restart-chaos/hosted-main.ts +++ b/tests/restart-chaos/hosted-main.ts @@ -43,11 +43,11 @@ export function keepPolling(workload: Pick, pollMs = POL } export async function finalizeHosted(input: { api: Parameters[0]["api"]; keep: boolean; label: string; dir: string; state: HostedState | null; - record: (kind: string, data: Record) => void }): Promise<{ errors: string[] } | null> { + record: (kind: string, data: Record) => void; terminatePollMs?: number }): Promise<{ errors: string[] } | null> { if (input.keep) return null; const state = input.state ?? await loadState(input.dir); if (!state) return null; - return await cleanupHosted({ api: input.api, label: input.label, companyId: state.companyId, state, record: input.record }) + return await cleanupHosted({ api: input.api, label: input.label, companyId: state.companyId, state, record: input.record, terminatePollMs: input.terminatePollMs }) .catch(error => ({ errors: [String(error)] })); } diff --git a/tests/restart-chaos/hosted-setup.ts b/tests/restart-chaos/hosted-setup.ts index 1235829b2f7..5a70f986187 100644 --- a/tests/restart-chaos/hosted-setup.ts +++ b/tests/restart-chaos/hosted-setup.ts @@ -163,7 +163,7 @@ async function removeAgent(api: SetupApi, agentId: string, pollMs: number): Prom if (current.status !== "terminated") await api.post(`/api/agents/${agentId}/terminate`); for (const until = Date.now() + TERMINATE_WAIT_MS; ;) { const agent = await api.get(`/api/agents/${agentId}`); - if (agent.status === "terminated") break; + if (agent.status === "terminated" && (agent.lifecycleState === undefined || agent.lifecycleState === null || agent.lifecycleState === "terminated")) break; if (Date.now() >= until) throw new Error(`agent still ${String(agent.status)} after ${TERMINATE_WAIT_MS / 1000} s; run --hosted-cleanup-only again`); await delay(pollMs); }