From 55228527cfe4f2b7dd69e5621bae2f1fd15fcba3 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 17 Nov 2026 04:07:23 -0600 Subject: [PATCH] test(restart-chaos): keep a run waiting on an active model request as its U1 owner (#21986) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The restart-chaos harness grades every task with universal checks. U1 checks that a task always has a live owner or an item a person can act on. > - U1 stops counting a `running` run as the owner after 3 minutes with no new run evidence. It does this even while the run's provider turn is still active. > - R5 in the same harness already allows an active provider turn 10 minutes of quiet (#21730), because one model request can be slow. > - In restart-chaos rerun 11 (task t001, OpenCode), one model request stayed quiet for 7.5 minutes. U1 failed the task, but the API reported the run `running` and the task `working` for the whole time, and the run then succeeded. > - This pull request makes U1 use the same provider-turn bound as R5. > - The benefit is that U1 no longer reports a slow model request as an unowned task, and it still fails a run that stays quiet for more than 10 minutes. ## Linked Issues or Issue Description Refs #21730 **What happened?** Restart-chaos rerun 11 A1 (master 22d3f9a03c, wave w-20261117T084017-s7106) graded t001 (CHA-1, OpenCode) as U1 `no_owner_no_action 183s in in_progress (interruption working)`, with `ownerlessMs` 182575, `queuedMs` 0 and `retryTimerMs` 0. The task had one run, `23ce0369`, from 08:47:10Z to 09:07:30Z, and it succeeded. All 32 tasks finished hands-off. **Evidence** - Run events: the agent's first `paperclip_finish` was refused at 08:58:58Z. The agent registered its deliverable at 08:59:21Z. The last streamed text arrived at 08:59:59Z (seq 348). The next event is the second `paperclip_finish` at 09:07:25Z (seq 349). No tool ran in between. - Observer samples: every sample from 08:59:59Z to 09:07:30Z shows the run `running`, `providerTurnActive: true`, the interruption state `working`, and the server ready. The two graceful hand-off swaps (08:50:42Z and 08:53:52Z) ended more than 5 minutes before this window, so the window is not a reattach gap. - U1 counted the run as unowned from 09:02:59Z (3 minutes after its last event). It failed at about 09:06:02Z, when the ownerless time passed 180 s. - R5 passed the same task, because R5 allows an active provider turn 10 minutes (`PROVIDER_TURN_LIMIT_MS`). **Expected behavior** U1 counts a quiet `running` run as the owner while its provider turn is active, up to the same 10-minute bound that R5 uses. **Product verdict** This is a grader misread, not a product gap. During the window the board showed the run as live and working, and the run then finished by itself. ## What Changed - `tests/restart-chaos/universal.ts`: `liveOwner` uses `PROVIDER_TURN_LIMIT_MS` (10 minutes) as the quiet limit for a `running` run with `providerTurnActive`. A preparing run keeps `PREPARATION_LIMIT_MS`. Other running runs keep `silentRunMs` (3 minutes). - `tests/restart-chaos/universal.test.ts`: a regression test in the t001 shape. A run that is quiet for 7.5 minutes during an active provider turn and then succeeds passes U1 with `ownerlessMs` 0. A run that stays quiet for 15 minutes still fails U1. - `tests/restart-chaos/README.md`: the R5 note says that U1 uses the same bounds, with the t001 evidence. ## Verification - `cd tests/restart-chaos && npx vitest run universal.test.ts`: 25 passed. - Red on master: with the master `universal.ts`, the new test fails (`expected { verdict: 'fail', … } to match object { verdict: 'pass', … }`). - No chaos wave was run for this change. Rerun 11 holds the slot. ## Risks - Low risk. The change affects only the harness grader. U1 now allows an active provider turn up to 10 minutes of quiet before it counts the task as unowned, which is the same bound that R5 already uses. A run that is quiet for longer still fails both checks. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, used through Claude Code with tool use (shell, file edits) for the evidence analysis, code and tests. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- tests/restart-chaos/README.md | 2 +- tests/restart-chaos/universal.test.ts | 11 +++++++++++ tests/restart-chaos/universal.ts | 5 +++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 925936fb915..c4c36f9900a 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -417,7 +417,7 @@ part of U3 is checked only in waves run with this harness. temporary instance is deleted. - Faults are serialized; each one hits every task running at that moment. - A `todo` or `in_progress` task with no live run and no named wait is not a rest. The observer keeps watching it until it resumes or passes the R5 stall limit. -- R5 also fails a run that stays `running` with no live evidence (no new run event, `lastUsefulActionAt` or `execution.lastConfirmedActivityAt`, and no start) for more than 3 minutes after the server is ready. A `running` status alone is not proof of life. A quiet run is still live while a tool it started has not ended (for example a 4-minute `bench.sh`) or while its recorded local runner process still runs with the same PID and start time. While a provider turn is active and no tool runs, one model request is outstanding: R5 allows up to 10 minutes of quiet before it counts a dead run. A slow OpenRouter request took 303.6 s in w-20261115T044649-s7106/t028, and common provider clients time out a request at 10 minutes. A run with no run event yet (`preparing`: its events read back empty) is still acquiring or preparing its environment, because "run started" is written only after that. R5 allows it 10 minutes too: in w-20261116T001158-s7106/t010 a Daytona restore of an archived sandbox took 251 s, including one 90 s plugin call timeout, and the Daytona provider bounds each start or restore at 5 minutes by default, so 10 minutes covers one timed-out restore and its retry. U1 uses the same bound (`PREPARATION_LIMIT_MS`) before it stops counting a preparing run as the task's owner. +- R5 also fails a run that stays `running` with no live evidence (no new run event, `lastUsefulActionAt` or `execution.lastConfirmedActivityAt`, and no start) for more than 3 minutes after the server is ready. A `running` status alone is not proof of life. A quiet run is still live while a tool it started has not ended (for example a 4-minute `bench.sh`) or while its recorded local runner process still runs with the same PID and start time. While a provider turn is active and no tool runs, one model request is outstanding: R5 allows up to 10 minutes of quiet before it counts a dead run. A slow OpenRouter request took 303.6 s in w-20261115T044649-s7106/t028, and common provider clients time out a request at 10 minutes. A run with no run event yet (`preparing`: its events read back empty) is still acquiring or preparing its environment, because "run started" is written only after that. R5 allows it 10 minutes too: in w-20261116T001158-s7106/t010 a Daytona restore of an archived sandbox took 251 s, including one 90 s plugin call timeout, and the Daytona provider bounds each start or restore at 5 minutes by default, so 10 minutes covers one timed-out restore and its retry. U1 uses the same bounds before it stops counting a running run as the task's owner: `PREPARATION_LIMIT_MS` for a preparing run and `PROVIDER_TURN_LIMIT_MS` while a provider turn is active. In w-20261117T084017-s7106/t001 one OpenCode model request stayed quiet for 7.5 minutes before the agent called `paperclip_finish`, while the API kept the run `running` and the task `working`. - The wave fingerprints the checkout and `packages/paperclip-runner/dist` at start and after every fault. A build, install or branch switch during a wave would make later restarts run different code, so it fails the wave as a harness error (`worktree_changed_mid_wave`). The baseline is taken after the harness's own setup (preflight rebuilds provider entrypoints) and covers the tracked diff, untracked file contents, the runner `dist` and the runnerd binary. Build before the wave, never during it. - `faults.json` and `task-faults.json` (each task's launch time and the faults that crossed it) are written after every fault, so `--replay` of a task in an interrupted or wedged wave, or a storm task, re-fires the faults that crossed it at the same offsets. - A fault that fails to complete fails the wave as a harness error (exit 2), even when recovery brings the server back. diff --git a/tests/restart-chaos/universal.test.ts b/tests/restart-chaos/universal.test.ts index ee85e466646..c8a3b884910 100644 --- a/tests/restart-chaos/universal.test.ts +++ b/tests/restart-chaos/universal.test.ts @@ -119,6 +119,17 @@ describe("U1 liveness", () => { const samples = polls(0, 7 * 60 * SEC, at => sample(at, "in_progress", [run("r1", "running", 0)])); expect(gradeU1(record(samples, { restAt: null }), BOUNDS).verdict).toBe("fail"); }); + + it("keeps a quiet run that waits on an active model request as its owner, within the provider-turn bound (rerun 11 t001)", () => { + const quiet = (untilMs: number) => [...polls(0, 60 * SEC, at => sample(at, "in_progress", [run("r1", "running", 0, null, + { lastEvidenceAt: iso(at), providerTurnActive: true })])), + ...polls(63 * SEC, untilMs, at => sample(at, "in_progress", [run("r1", "running", 0, null, + { lastEvidenceAt: iso(60 * SEC), providerTurnActive: true })], { interruption: { ...retryable, state: "working", canRetry: false } }))]; + const waited = [...quiet(510 * SEC), sample(513 * SEC, "done", [run("r1", "succeeded", 0, 512 * SEC)])]; + expect(gradeU1(record(waited), BOUNDS)).toMatchObject({ verdict: "pass", metrics: { ownerlessMs: 0 } }); + expect(gradeU1(record(quiet(15 * 60 * SEC), { restAt: null }), BOUNDS)).toMatchObject({ verdict: "fail", + detail: expect.stringMatching(/^no_owner_no_action .* in in_progress \(interruption working\)/) }); + }); }); describe("U2 retry loops", () => { diff --git a/tests/restart-chaos/universal.ts b/tests/restart-chaos/universal.ts index 99ff4947e11..9aa9d8f1350 100644 --- a/tests/restart-chaos/universal.ts +++ b/tests/restart-chaos/universal.ts @@ -12,7 +12,7 @@ * (reported in the grade and ledger, never fails the task). The split comes * from the false-positive audit over saved waves; see README "Universal checks". */ -import { PREPARATION_LIMIT_MS } from "./grader.js"; +import { PREPARATION_LIMIT_MS, PROVIDER_TURN_LIMIT_MS } from "./grader.js"; import type { RunSample, TaskRecord, TaskSample } from "./types.js"; export type UniversalId = "U1" | "U2" | "U3" | "U4" | "U5"; @@ -138,7 +138,8 @@ export function liveOwner(sample: TaskSample, queuedFor: ReadonlyMap