From 0502cc9facf46b8641455b1af88e4a9146a0b8ca Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:12:03 -0600 Subject: [PATCH] test(chaos): retry a blocked M1 subtask and check the parent's stall wake (#20147) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The restart chaos harness grades the M1 scenario: a parent task delegates two subtasks across restarts > - When a subtask's run failed recovery, the harness only clicked Retry on the parent. The parent's Retry is correctly held behind the blocked subtask (#20109), so the task waited until its deadline > - A real board user would also click the subtask's Retry. #20098 also added a wake for a parent whose subtask stalls, and the grade did not check it > - This pull request clicks the board Retry on a blocked subtask (graded as a needed Retry) and checks that the parent got exactly one stall wake per stall > - The benefit is that M1 grades the full user path and proves the stall wake from the product's own activity record ## Linked Issues or Issue Description Refs #20098, #20109, #20094 **What happened?** In wave `w-20261109T003126-s9701`, the M1 parent t05 rested blocked behind a blocked codex subtask. The harness clicked Retry on the parent. The product held that Retry ("waiting on a blocked task") as designed. Nothing retried the subtask, and the task waited 38 minutes until the wave deadline. The wave also could not show whether the parent got the new `issue_blocker_stalled` wake. **Expected behavior** The harness acts like the board: it clicks Retry on a blocked subtask that offers one, and it counts that as a needed Retry. The M1 oracle checks the parent's activity for one `issue.blocker_stalled_wake` per stalled subtask. ## What Changed - `observer.ts`: - For M1, the observer checks the parent's subtasks every 10 s while the parent is open, independent of the parent's rest window. - Each blocked subtask is recorded as a stall: the subtask plus its latest recovery action, from `GET /api/issues/:id/recovery-actions`. - When a blocked subtask's interruption offers Retry, the harness posts the board Retry. It clicks once per subtask; a click that was not accepted can be tried again on a later check. - The parent's `issue.blocker_stalled_wake` activity entries are collected as `stallWakes` through `GET /api/issues/:id/activity`. - `grader.ts`: R3 fails with `retry_needed_on_subtask_ ()` when a subtask needed a board Retry, unless the parent's own Retry already failed R3. - `workloads.ts`: the new M1 check `parent_woken_once_per_stall` is added when a subtask stalled. Each (subtask, recovery action) stall must have exactly one matching wake. A stall with an unknown action matches any wake for that subtask. ## Verification - `npx vitest run` in `tests/restart-chaos`: 203 passed. New tests cover: - an observer test: a blocked subtask gets one Retry click, and the parent is still watched; - the R3 subtask-retry grade; - the stall-wake check: woken once, not woken, woken twice, two stalls of one subtask with only one woken, and no stall. - G1, which also has subtasks, is unaffected: the existing governance tests pass. - `node scripts/typecheck-test-projects.mjs tests/restart-chaos`: clean. ## Risks - The harness now acts on subtasks, which it did not before. It clicks once per subtask and only when the product offers Retry. - The stall-wake check reads activity through the board API. If the API omits these entries for a board user, the check fails visibly instead of passing. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- tests/restart-chaos/grader.test.ts | 19 ++++++++- tests/restart-chaos/grader.ts | 7 +++ tests/restart-chaos/observer.test.ts | 30 +++++++++++++ tests/restart-chaos/observer.ts | 64 +++++++++++++++++++++++++++- tests/restart-chaos/types.ts | 4 ++ tests/restart-chaos/workloads.ts | 15 +++++++ 6 files changed, 137 insertions(+), 2 deletions(-) diff --git a/tests/restart-chaos/grader.test.ts b/tests/restart-chaos/grader.test.ts index ad72b13e9c4..1f133a67178 100644 --- a/tests/restart-chaos/grader.test.ts +++ b/tests/restart-chaos/grader.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { crossingFaults, gradeR3, gradeR4, gradeR5, gradeTask, STALL_LIMIT_MS } from "./grader.js"; import { isResting } from "./observer.js"; import type { FaultReceipt, OracleArtifacts, RunSample, TaskRecord, TaskSample } from "./types.js"; -import { BENCH_EXPECTED, parseBenchRecord, r1TestChecksSlug, workloadDefinitions } from "./workloads.js"; +import { BENCH_EXPECTED, parseBenchRecord, r1TestChecksSlug, stallWakeChecks, workloadDefinitions } from "./workloads.js"; const T0 = Date.parse("2026-11-06T12:00:00Z"); const iso = (offsetMs: number) => new Date(T0 + offsetMs).toISOString(); @@ -221,6 +221,23 @@ describe("T-oracles", () => { expect(workloadDefinitions.C1.grade("abcd1234", { ...passingBrief, documents: { brief: `${passingBrief.documents.brief}FOLLOWUP-abcd1234\n` } }).verdict).toBe("pass"); }); + it("fails M1 when a subtask needed a board Retry, and checks the parent got one stall wake per stall", () => { + const done = record([sample(1_000, "done", [run("r1", "succeeded", 0, 900)])], { finalStatus: "done" }); + const withChildRetry = { ...done, childRetries: [{ childIssueId: "c2", at: 1, observedRunId: "r", httpStatus: 200, accepted: true, error: null }] }; + expect(gradeR3(done)).toMatchObject({ verdict: "n/a" }); + expect(gradeR3(withChildRetry)).toMatchObject({ verdict: "fail", detail: "retry_needed_on_subtask_1 (c2)" }); + const stalled = { issueStatus: "done", documents: {}, agentComments: [], files: {}, commands: {}, workspacePath: null, + childStalls: [{ childIssueId: "c2", recoveryActionId: "a1" }] }; + const wakeA1 = { stalledIssueId: "c2", recoveryActionId: "a1" }; + expect(stallWakeChecks({ ...stalled, stallWakes: [wakeA1] } as never)[0]).toMatchObject({ ok: true }); + expect(stallWakeChecks({ ...stalled, stallWakes: [] } as never)[0]).toMatchObject({ ok: false, detail: "stalls=1 wakes=0 unwoken=c2:a1" }); + expect(stallWakeChecks({ ...stalled, stallWakes: [wakeA1, wakeA1] } as never)[0]).toMatchObject({ ok: false }); + const twoStalls = { ...stalled, childStalls: [...stalled.childStalls, { childIssueId: "c2", recoveryActionId: "a2" }] }; + expect(stallWakeChecks({ ...twoStalls, stallWakes: [wakeA1] } as never)[0]).toMatchObject({ ok: false, detail: "stalls=2 wakes=1 unwoken=c2:a2" }); + expect(stallWakeChecks({ ...twoStalls, stallWakes: [wakeA1, { stalledIssueId: "c2", recoveryActionId: "a2" }] } as never)[0]).toMatchObject({ ok: true }); + expect(stallWakeChecks({ ...stalled, childStalls: [] } as never)).toEqual([]); + }); + it("grades B1 by running the tests and the CLI", () => { const base: OracleArtifacts = { issueStatus: "done", documents: {}, agentComments: [], workspacePath: "/tmp/ws", files: { "NOTES-abcd1234.md": "B1-abcd1234\n" }, diff --git a/tests/restart-chaos/grader.ts b/tests/restart-chaos/grader.ts index 822a905059d..33963debf17 100644 --- a/tests/restart-chaos/grader.ts +++ b/tests/restart-chaos/grader.ts @@ -93,6 +93,13 @@ export function gradeR2(record: TaskRecord, oracle: OracleResult): RuleResult { } export function gradeR3(record: TaskRecord): RuleResult { + const own = gradeOwnRetry(record); + const childClicks = record.childRetries?.length ?? 0; + if (!childClicks || own.verdict === "fail") return own; + return { verdict: "fail", detail: `retry_needed_on_subtask_${childClicks} (${record.childRetries!.map(attempt => attempt.childIssueId).join(",")})` }; +} + +function gradeOwnRetry(record: TaskRecord): RuleResult { const last = lastReachable(record); if (!record.retry) { if (record.finalStatus !== expectedFinalStatus(record) && !record.timedOut && !record.launchError) { diff --git a/tests/restart-chaos/observer.test.ts b/tests/restart-chaos/observer.test.ts index 51ad4916034..1ea8d6b37da 100644 --- a/tests/restart-chaos/observer.test.ts +++ b/tests/restart-chaos/observer.test.ts @@ -108,6 +108,36 @@ describe("TaskObserver", () => { expect(observer.record.finalStatus).toBe("blocked"); }); + it("clicks the board Retry on a blocked subtask, as a user would, and keeps watching the parent", async () => { + let childStatus = "blocked"; + const base = fakeApi({ "parentIds=": () => [{ id: "c1", parentId: "i1", status: "done" }, { id: "c2", parentId: "i1", status: childStatus }], + "/documents/": () => null, "/recovery-actions": () => ({ active: { id: "a1" }, actions: [] }), "/api/issues/c2": () => ({ id: "c2", status: childStatus, + interruption: { canRetry: childStatus === "blocked", observedRunId: "child-run" } }) }, "blocked"); + const childRetries: string[] = []; + const api = { ...base.api, get: base.api.get.bind(base.api), + async post(route: string, body: unknown) { + if (route === "/api/issues/c2/retry") { childRetries.push(route); childStatus = "in_progress"; return { accepted: true }; } + return (base.api as unknown as { post(route: string, body: unknown): Promise }).post(route, body); + } } as unknown as ChaosApi; + const timeline = new Timeline(null); + const observer = new TaskObserver({ ...plan, workload: "M1", followUpAfterMs: null }, api, lab, timeline, ready); + await observer.launch(); + let now = observer.record.createdAt! + 1_000; + const restWindow = async () => { await observer.poll(now); now += REST_CONFIRM_MS + 1; await observer.poll(now); now += 1_000; }; + await restWindow(); + expect(childRetries).toEqual(["/api/issues/c2/retry"]); + expect(observer.record.childRetries).toMatchObject([{ childIssueId: "c2", accepted: true, observedRunId: "child-run" }]); + expect(observer.record.childStalls).toEqual([{ childIssueId: "c2", recoveryActionId: "a1" }]); + now += CHILD_WAKE_GRACE_MS; + await restWindow(); + expect(observer.done).toBe(false); + childStatus = "done"; + await restWindow(); + now += CHILD_WAKE_GRACE_MS; + await restWindow(); + expect(childRetries).toHaveLength(1); + }); + it("closes an open task as unfinished at interrupt, so the wave grades it as a failure", async () => { const { api } = fakeApi({}, "in_progress"); const timeline = new Timeline(null); diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index 61483f1ed31..efd5dd47e70 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -49,6 +49,7 @@ const RESULT_TRIGGER_EVENTS = new Set(["run.result.proposed", "run.result.accept /** Rest must hold for this long before the observer acts on it. */ export const REST_CONFIRM_MS = 12_000; export const CHILD_WAKE_GRACE_MS = 60_000; +export const CHILD_STALL_CHECK_MS = 10_000; export function childrenPending(children: ReadonlyArray<{ status: string | null }> | null): boolean { return !children || children.some(child => !["done", "cancelled", "blocked"].includes(child.status ?? "")); @@ -167,6 +168,7 @@ export class TaskObserver { issue: Json | null = null; private restCandidateSince: number | null = null; private childrenSettledAt: number | null = null; + private childStallCheckedAt: number | null = null; private stallSince: number | null = null; /** First sample of the current stretch with the server ready (H1 holds the pause across it). */ private readySince: number | null = null; @@ -409,6 +411,10 @@ export class TaskObserver { this.readySince = sample.reachable && sample.serverReady ? (this.readySince ?? now) : null; await this.observeBenchProgress(); await this.governanceStep(sample, now); + if (await this.observeChildStalls(now)) { + this.restCandidateSince = null; + this.childrenSettledAt = null; + } if (await this.maybeFollowUp(sample, now)) { // The comment response is not the agent's answer: require fresh samples. this.restCandidateSince = null; @@ -540,6 +546,61 @@ export class TaskObserver { } /** Exactly one click, with the same body ui/src/hooks/useTaskRetry.ts sends. */ + private async observeChildStalls(now: number): Promise { + if (this.plan.workload !== "M1" || !this.record.issueId) return false; + if (this.childStallCheckedAt !== null && now - this.childStallCheckedAt < CHILD_STALL_CHECK_MS) return false; + this.childStallCheckedAt = now; + const rows = await this.api.get(`/api/companies/${this.lab.companyId}/issues?view=index&parentIds=${encodeURIComponent(this.record.issueId)}&limit=20`) + .catch(() => null); + const blocked = (Array.isArray(rows) ? rows : []).filter(row => row.parentId === this.record.issueId && row.status === "blocked").map(row => String(row.id)); + for (const childIssueId of blocked) { + const recovery = await this.api.get(`/api/issues/${childIssueId}/recovery-actions`).catch(() => null); + const actions = Array.isArray(recovery?.actions) ? recovery.actions as Json[] : []; + const latest = (recovery?.active as Json | null | undefined) ?? actions[0] ?? null; + const recoveryActionId = typeof latest?.id === "string" ? latest.id : null; + const stalls = this.record.childStalls ??= []; + if (!stalls.some(stall => stall.childIssueId === childIssueId && stall.recoveryActionId === recoveryActionId)) { + stalls.push({ childIssueId, recoveryActionId }); + this.timeline.add("task_child_stalled", { taskId: this.plan.taskId, childIssueId, recoveryActionId }); + } + } + return this.retryBlockedChildren(blocked, now); + } + + private async retryBlockedChildren(childIds: readonly string[], now: number): Promise { + let clicked = false; + for (const childIssueId of childIds) { + if ((this.record.childRetries ?? []).some(attempt => attempt.childIssueId === childIssueId && attempt.accepted)) continue; + const child = await this.api.get(`/api/issues/${childIssueId}`).catch(() => null); + const interruption = child?.interruption as Json | null | undefined; + if (interruption?.canRetry !== true) continue; + const observedRunId = typeof interruption.observedRunId === "string" ? interruption.observedRunId : null; + const attempt: RetryAttempt & { childIssueId: string } = { childIssueId, at: now, observedRunId, httpStatus: null, accepted: null, error: null }; + try { + const result = await this.api.post<{ accepted?: boolean }>(`/api/issues/${childIssueId}/retry`, { idempotencyKey: randomUUID(), observedRunId }); + attempt.httpStatus = 200; + attempt.accepted = result?.accepted === true; + attempt.acceptedAt = Math.max(now, Date.now()); + } catch (error) { + attempt.httpStatus = error instanceof ChaosApiError ? error.status : null; + attempt.error = error instanceof Error ? error.message : String(error); + } + (this.record.childRetries ??= []).push(attempt); + this.timeline.add("task_child_retry_clicked", { taskId: this.plan.taskId, ...attempt }); + clicked = clicked || attempt.accepted === true; + } + return clicked; + } + + private async collectStallWakes(issueId: string): Promise> { + const rows = await this.api.get(`/api/issues/${issueId}/activity`).catch(() => []); + return (rows ?? []).filter(row => row.action === "issue.blocker_stalled_wake").map(row => { + const details = (row.details ?? {}) as Json; + return { stalledIssueId: typeof details.stalledIssueId === "string" ? details.stalledIssueId : null, + recoveryActionId: typeof details.recoveryActionId === "string" ? details.recoveryActionId : null }; + }); + } + private async retry(sample: TaskSample, now: number): Promise { const observedRunId = sample.interruption?.observedRunId ?? (this.issue?.executionBlocker?.runId as string | undefined) ?? null; const attempt: RetryAttempt = { at: now, observedRunId, httpStatus: null, accepted: null, error: null }; @@ -770,7 +831,8 @@ export class TaskObserver { agentComments: (comments ?? []).filter(comment => comment.authorAgentId).map(comment => String(comment.body ?? "")), files, commands, workspacePath, ...(this.plan.workload === "B5" ? { benchProgressObserved: this.record.benchProgressSeen?.length ?? 0 } : {}), - ...(definition.collect.childDocuments ? { children: await this.collectChildren(issueId) } : {}), + ...(definition.collect.childDocuments ? { children: await this.collectChildren(issueId), + childStalls: this.record.childStalls ?? [], stallWakes: await this.collectStallWakes(issueId) } : {}), ...(this.plan.workload === "G1" ? { approvals: await this.collectApprovals(issueId) } : {}), ...(this.plan.workload === "H1" ? await this.finalPauseCheck(issueId) : {}), ...(this.record.governance ? { governance: this.record.governance } : {}), diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index 427985f2b6c..fb02f8a617f 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -222,6 +222,8 @@ export interface OracleArtifacts { /** B5: distinct `.bench/progress` values the harness itself saw while the task was live. */ benchProgressObserved?: number; children?: Array<{ id: string; title?: string | null; status: string | null; assigneeAgentId: string | null; documents: Record }>; + childStalls?: Array<{ childIssueId: string; recoveryActionId: string | null }>; + stallWakes?: Array<{ stalledIssueId: string | null; recoveryActionId: string | null }>; /** G1: approvals linked to the task. */ approvals?: Array<{ id: string; status: string; title: string | null }>; /** G1 and H1: the board's actions and what the observer saw around them. */ @@ -288,6 +290,8 @@ export interface TaskRecord { lastFaultAt?: number; restSnapshotError?: string; priorRetries?: RetryAttempt[]; + childStalls?: Array<{ childIssueId: string; recoveryActionId: string | null }>; + childRetries?: Array; workspaceMovedTo?: string; /** Set when the pre-fault snapshot could not read the workspace; R2 then fails visibly. */ preFaultSnapshotError?: string | null; diff --git a/tests/restart-chaos/workloads.ts b/tests/restart-chaos/workloads.ts index c1c837191f0..46693c100d6 100644 --- a/tests/restart-chaos/workloads.ts +++ b/tests/restart-chaos/workloads.ts @@ -60,6 +60,20 @@ export function parseBenchRecord(text: string | null | undefined) { ? { seconds: finished - started, result: fields.result } : null; } +export function stallWakeChecks(artifacts: OracleArtifacts): OracleResult["checks"] { + const stalls = artifacts.childStalls ?? []; + if (!stalls.length) return []; + const wakes = artifacts.stallWakes ?? []; + const matching = (stall: { childIssueId: string; recoveryActionId: string | null }) => wakes.filter(wake => + wake.stalledIssueId === stall.childIssueId && (stall.recoveryActionId === null || wake.recoveryActionId === stall.recoveryActionId)); + const unwoken = stalls.filter(stall => matching(stall).length === 0).map(stall => `${stall.childIssueId}:${stall.recoveryActionId}`); + const perPair = new Map(); + for (const wake of wakes) perPair.set(`${wake.stalledIssueId}:${wake.recoveryActionId}`, (perPair.get(`${wake.stalledIssueId}:${wake.recoveryActionId}`) ?? 0) + 1); + const repeated = [...perPair.entries()].filter(([, count]) => count > 1).map(([key]) => key); + return [{ name: "parent_woken_once_per_stall", ok: !unwoken.length && !repeated.length, + detail: `stalls=${stalls.length} wakes=${wakes.length}${unwoken.length ? ` unwoken=${unwoken.join(",")}` : ""}${repeated.length ? ` repeated=${repeated.join(",")}` : ""}` }]; +} + function result(checks: OracleResult["checks"]): OracleResult { return { verdict: checks.every(check => check.ok) ? "pass" : "fail", checks }; } @@ -205,6 +219,7 @@ export const workloadDefinitions: Record = { { name: "part2_kept", ok: distinct && Boolean(agents) && part2[0]!.assigneeAgentId === agents!.self.id, detail: `assignee=${part2[0]?.assigneeAgentId ?? "none"}` }, { name: "summary_has_both_parts", ok: summary.includes(partMarker(nonce, 1)) && summary.includes(partMarker(nonce, 2)) }, + ...stallWakeChecks(artifacts), ]); }, },