diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 925936fb915..c4c36f9900a 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -417,7 +417,7 @@ part of U3 is checked only in waves run with this harness. temporary instance is deleted. - Faults are serialized; each one hits every task running at that moment. - A `todo` or `in_progress` task with no live run and no named wait is not a rest. The observer keeps watching it until it resumes or passes the R5 stall limit. -- R5 also fails a run that stays `running` with no live evidence (no new run event, `lastUsefulActionAt` or `execution.lastConfirmedActivityAt`, and no start) for more than 3 minutes after the server is ready. A `running` status alone is not proof of life. A quiet run is still live while a tool it started has not ended (for example a 4-minute `bench.sh`) or while its recorded local runner process still runs with the same PID and start time. While a provider turn is active and no tool runs, one model request is outstanding: R5 allows up to 10 minutes of quiet before it counts a dead run. A slow OpenRouter request took 303.6 s in w-20261115T044649-s7106/t028, and common provider clients time out a request at 10 minutes. A run with no run event yet (`preparing`: its events read back empty) is still acquiring or preparing its environment, because "run started" is written only after that. R5 allows it 10 minutes too: in w-20261116T001158-s7106/t010 a Daytona restore of an archived sandbox took 251 s, including one 90 s plugin call timeout, and the Daytona provider bounds each start or restore at 5 minutes by default, so 10 minutes covers one timed-out restore and its retry. U1 uses the same bound (`PREPARATION_LIMIT_MS`) before it stops counting a preparing run as the task's owner. +- R5 also fails a run that stays `running` with no live evidence (no new run event, `lastUsefulActionAt` or `execution.lastConfirmedActivityAt`, and no start) for more than 3 minutes after the server is ready. A `running` status alone is not proof of life. A quiet run is still live while a tool it started has not ended (for example a 4-minute `bench.sh`) or while its recorded local runner process still runs with the same PID and start time. While a provider turn is active and no tool runs, one model request is outstanding: R5 allows up to 10 minutes of quiet before it counts a dead run. A slow OpenRouter request took 303.6 s in w-20261115T044649-s7106/t028, and common provider clients time out a request at 10 minutes. A run with no run event yet (`preparing`: its events read back empty) is still acquiring or preparing its environment, because "run started" is written only after that. R5 allows it 10 minutes too: in w-20261116T001158-s7106/t010 a Daytona restore of an archived sandbox took 251 s, including one 90 s plugin call timeout, and the Daytona provider bounds each start or restore at 5 minutes by default, so 10 minutes covers one timed-out restore and its retry. U1 uses the same bounds before it stops counting a running run as the task's owner: `PREPARATION_LIMIT_MS` for a preparing run and `PROVIDER_TURN_LIMIT_MS` while a provider turn is active. In w-20261117T084017-s7106/t001 one OpenCode model request stayed quiet for 7.5 minutes before the agent called `paperclip_finish`, while the API kept the run `running` and the task `working`. - The wave fingerprints the checkout and `packages/paperclip-runner/dist` at start and after every fault. A build, install or branch switch during a wave would make later restarts run different code, so it fails the wave as a harness error (`worktree_changed_mid_wave`). The baseline is taken after the harness's own setup (preflight rebuilds provider entrypoints) and covers the tracked diff, untracked file contents, the runner `dist` and the runnerd binary. Build before the wave, never during it. - `faults.json` and `task-faults.json` (each task's launch time and the faults that crossed it) are written after every fault, so `--replay` of a task in an interrupted or wedged wave, or a storm task, re-fires the faults that crossed it at the same offsets. - A fault that fails to complete fails the wave as a harness error (exit 2), even when recovery brings the server back. diff --git a/tests/restart-chaos/universal.test.ts b/tests/restart-chaos/universal.test.ts index ee85e466646..c8a3b884910 100644 --- a/tests/restart-chaos/universal.test.ts +++ b/tests/restart-chaos/universal.test.ts @@ -119,6 +119,17 @@ describe("U1 liveness", () => { const samples = polls(0, 7 * 60 * SEC, at => sample(at, "in_progress", [run("r1", "running", 0)])); expect(gradeU1(record(samples, { restAt: null }), BOUNDS).verdict).toBe("fail"); }); + + it("keeps a quiet run that waits on an active model request as its owner, within the provider-turn bound (rerun 11 t001)", () => { + const quiet = (untilMs: number) => [...polls(0, 60 * SEC, at => sample(at, "in_progress", [run("r1", "running", 0, null, + { lastEvidenceAt: iso(at), providerTurnActive: true })])), + ...polls(63 * SEC, untilMs, at => sample(at, "in_progress", [run("r1", "running", 0, null, + { lastEvidenceAt: iso(60 * SEC), providerTurnActive: true })], { interruption: { ...retryable, state: "working", canRetry: false } }))]; + const waited = [...quiet(510 * SEC), sample(513 * SEC, "done", [run("r1", "succeeded", 0, 512 * SEC)])]; + expect(gradeU1(record(waited), BOUNDS)).toMatchObject({ verdict: "pass", metrics: { ownerlessMs: 0 } }); + expect(gradeU1(record(quiet(15 * 60 * SEC), { restAt: null }), BOUNDS)).toMatchObject({ verdict: "fail", + detail: expect.stringMatching(/^no_owner_no_action .* in in_progress \(interruption working\)/) }); + }); }); describe("U2 retry loops", () => { diff --git a/tests/restart-chaos/universal.ts b/tests/restart-chaos/universal.ts index 99ff4947e11..9aa9d8f1350 100644 --- a/tests/restart-chaos/universal.ts +++ b/tests/restart-chaos/universal.ts @@ -12,7 +12,7 @@ * (reported in the grade and ledger, never fails the task). The split comes * from the false-positive audit over saved waves; see README "Universal checks". */ -import { PREPARATION_LIMIT_MS } from "./grader.js"; +import { PREPARATION_LIMIT_MS, PROVIDER_TURN_LIMIT_MS } from "./grader.js"; import type { RunSample, TaskRecord, TaskSample } from "./types.js"; export type UniversalId = "U1" | "U2" | "U3" | "U4" | "U5"; @@ -138,7 +138,8 @@ export function liveOwner(sample: TaskSample, queuedFor: ReadonlyMap