diff --git a/tests/restart-chaos/grader.test.ts b/tests/restart-chaos/grader.test.ts index ad72b13e9c4..1f133a67178 100644 --- a/tests/restart-chaos/grader.test.ts +++ b/tests/restart-chaos/grader.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { crossingFaults, gradeR3, gradeR4, gradeR5, gradeTask, STALL_LIMIT_MS } from "./grader.js"; import { isResting } from "./observer.js"; import type { FaultReceipt, OracleArtifacts, RunSample, TaskRecord, TaskSample } from "./types.js"; -import { BENCH_EXPECTED, parseBenchRecord, r1TestChecksSlug, workloadDefinitions } from "./workloads.js"; +import { BENCH_EXPECTED, parseBenchRecord, r1TestChecksSlug, stallWakeChecks, workloadDefinitions } from "./workloads.js"; const T0 = Date.parse("2026-11-06T12:00:00Z"); const iso = (offsetMs: number) => new Date(T0 + offsetMs).toISOString(); @@ -221,6 +221,23 @@ describe("T-oracles", () => { expect(workloadDefinitions.C1.grade("abcd1234", { ...passingBrief, documents: { brief: `${passingBrief.documents.brief}FOLLOWUP-abcd1234\n` } }).verdict).toBe("pass"); }); + it("fails M1 when a subtask needed a board Retry, and checks the parent got one stall wake per stall", () => { + const done = record([sample(1_000, "done", [run("r1", "succeeded", 0, 900)])], { finalStatus: "done" }); + const withChildRetry = { ...done, childRetries: [{ childIssueId: "c2", at: 1, observedRunId: "r", httpStatus: 200, accepted: true, error: null }] }; + expect(gradeR3(done)).toMatchObject({ verdict: "n/a" }); + expect(gradeR3(withChildRetry)).toMatchObject({ verdict: "fail", detail: "retry_needed_on_subtask_1 (c2)" }); + const stalled = { issueStatus: "done", documents: {}, agentComments: [], files: {}, commands: {}, workspacePath: null, + childStalls: [{ childIssueId: "c2", recoveryActionId: "a1" }] }; + const wakeA1 = { stalledIssueId: "c2", recoveryActionId: "a1" }; + expect(stallWakeChecks({ ...stalled, stallWakes: [wakeA1] } as never)[0]).toMatchObject({ ok: true }); + expect(stallWakeChecks({ ...stalled, stallWakes: [] } as never)[0]).toMatchObject({ ok: false, detail: "stalls=1 wakes=0 unwoken=c2:a1" }); + expect(stallWakeChecks({ ...stalled, stallWakes: [wakeA1, wakeA1] } as never)[0]).toMatchObject({ ok: false }); + const twoStalls = { ...stalled, childStalls: [...stalled.childStalls, { childIssueId: "c2", recoveryActionId: "a2" }] }; + expect(stallWakeChecks({ ...twoStalls, stallWakes: [wakeA1] } as never)[0]).toMatchObject({ ok: false, detail: "stalls=2 wakes=1 unwoken=c2:a2" }); + expect(stallWakeChecks({ ...twoStalls, stallWakes: [wakeA1, { stalledIssueId: "c2", recoveryActionId: "a2" }] } as never)[0]).toMatchObject({ ok: true }); + expect(stallWakeChecks({ ...stalled, childStalls: [] } as never)).toEqual([]); + }); + it("grades B1 by running the tests and the CLI", () => { const base: OracleArtifacts = { issueStatus: "done", documents: {}, agentComments: [], workspacePath: "/tmp/ws", files: { "NOTES-abcd1234.md": "B1-abcd1234\n" }, diff --git a/tests/restart-chaos/grader.ts b/tests/restart-chaos/grader.ts index 822a905059d..33963debf17 100644 --- a/tests/restart-chaos/grader.ts +++ b/tests/restart-chaos/grader.ts @@ -93,6 +93,13 @@ export function gradeR2(record: TaskRecord, oracle: OracleResult): RuleResult { } export function gradeR3(record: TaskRecord): RuleResult { + const own = gradeOwnRetry(record); + const childClicks = record.childRetries?.length ?? 0; + if (!childClicks || own.verdict === "fail") return own; + return { verdict: "fail", detail: `retry_needed_on_subtask_${childClicks} (${record.childRetries!.map(attempt => attempt.childIssueId).join(",")})` }; +} + +function gradeOwnRetry(record: TaskRecord): RuleResult { const last = lastReachable(record); if (!record.retry) { if (record.finalStatus !== expectedFinalStatus(record) && !record.timedOut && !record.launchError) { diff --git a/tests/restart-chaos/observer.test.ts b/tests/restart-chaos/observer.test.ts index 51ad4916034..1ea8d6b37da 100644 --- a/tests/restart-chaos/observer.test.ts +++ b/tests/restart-chaos/observer.test.ts @@ -108,6 +108,36 @@ describe("TaskObserver", () => { expect(observer.record.finalStatus).toBe("blocked"); }); + it("clicks the board Retry on a blocked subtask, as a user would, and keeps watching the parent", async () => { + let childStatus = "blocked"; + const base = fakeApi({ "parentIds=": () => [{ id: "c1", parentId: "i1", status: "done" }, { id: "c2", parentId: "i1", status: childStatus }], + "/documents/": () => null, "/recovery-actions": () => ({ active: { id: "a1" }, actions: [] }), "/api/issues/c2": () => ({ id: "c2", status: childStatus, + interruption: { canRetry: childStatus === "blocked", observedRunId: "child-run" } }) }, "blocked"); + const childRetries: string[] = []; + const api = { ...base.api, get: base.api.get.bind(base.api), + async post(route: string, body: unknown) { + if (route === "/api/issues/c2/retry") { childRetries.push(route); childStatus = "in_progress"; return { accepted: true }; } + return (base.api as unknown as { post(route: string, body: unknown): Promise }).post(route, body); + } } as unknown as ChaosApi; + const timeline = new Timeline(null); + const observer = new TaskObserver({ ...plan, workload: "M1", followUpAfterMs: null }, api, lab, timeline, ready); + await observer.launch(); + let now = observer.record.createdAt! + 1_000; + const restWindow = async () => { await observer.poll(now); now += REST_CONFIRM_MS + 1; await observer.poll(now); now += 1_000; }; + await restWindow(); + expect(childRetries).toEqual(["/api/issues/c2/retry"]); + expect(observer.record.childRetries).toMatchObject([{ childIssueId: "c2", accepted: true, observedRunId: "child-run" }]); + expect(observer.record.childStalls).toEqual([{ childIssueId: "c2", recoveryActionId: "a1" }]); + now += CHILD_WAKE_GRACE_MS; + await restWindow(); + expect(observer.done).toBe(false); + childStatus = "done"; + await restWindow(); + now += CHILD_WAKE_GRACE_MS; + await restWindow(); + expect(childRetries).toHaveLength(1); + }); + it("closes an open task as unfinished at interrupt, so the wave grades it as a failure", async () => { const { api } = fakeApi({}, "in_progress"); const timeline = new Timeline(null); diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index 61483f1ed31..efd5dd47e70 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -49,6 +49,7 @@ const RESULT_TRIGGER_EVENTS = new Set(["run.result.proposed", "run.result.accept /** Rest must hold for this long before the observer acts on it. */ export const REST_CONFIRM_MS = 12_000; export const CHILD_WAKE_GRACE_MS = 60_000; +export const CHILD_STALL_CHECK_MS = 10_000; export function childrenPending(children: ReadonlyArray<{ status: string | null }> | null): boolean { return !children || children.some(child => !["done", "cancelled", "blocked"].includes(child.status ?? "")); @@ -167,6 +168,7 @@ export class TaskObserver { issue: Json | null = null; private restCandidateSince: number | null = null; private childrenSettledAt: number | null = null; + private childStallCheckedAt: number | null = null; private stallSince: number | null = null; /** First sample of the current stretch with the server ready (H1 holds the pause across it). */ private readySince: number | null = null; @@ -409,6 +411,10 @@ export class TaskObserver { this.readySince = sample.reachable && sample.serverReady ? (this.readySince ?? now) : null; await this.observeBenchProgress(); await this.governanceStep(sample, now); + if (await this.observeChildStalls(now)) { + this.restCandidateSince = null; + this.childrenSettledAt = null; + } if (await this.maybeFollowUp(sample, now)) { // The comment response is not the agent's answer: require fresh samples. this.restCandidateSince = null; @@ -540,6 +546,61 @@ export class TaskObserver { } /** Exactly one click, with the same body ui/src/hooks/useTaskRetry.ts sends. */ + private async observeChildStalls(now: number): Promise { + if (this.plan.workload !== "M1" || !this.record.issueId) return false; + if (this.childStallCheckedAt !== null && now - this.childStallCheckedAt < CHILD_STALL_CHECK_MS) return false; + this.childStallCheckedAt = now; + const rows = await this.api.get(`/api/companies/${this.lab.companyId}/issues?view=index&parentIds=${encodeURIComponent(this.record.issueId)}&limit=20`) + .catch(() => null); + const blocked = (Array.isArray(rows) ? rows : []).filter(row => row.parentId === this.record.issueId && row.status === "blocked").map(row => String(row.id)); + for (const childIssueId of blocked) { + const recovery = await this.api.get(`/api/issues/${childIssueId}/recovery-actions`).catch(() => null); + const actions = Array.isArray(recovery?.actions) ? recovery.actions as Json[] : []; + const latest = (recovery?.active as Json | null | undefined) ?? actions[0] ?? null; + const recoveryActionId = typeof latest?.id === "string" ? latest.id : null; + const stalls = this.record.childStalls ??= []; + if (!stalls.some(stall => stall.childIssueId === childIssueId && stall.recoveryActionId === recoveryActionId)) { + stalls.push({ childIssueId, recoveryActionId }); + this.timeline.add("task_child_stalled", { taskId: this.plan.taskId, childIssueId, recoveryActionId }); + } + } + return this.retryBlockedChildren(blocked, now); + } + + private async retryBlockedChildren(childIds: readonly string[], now: number): Promise { + let clicked = false; + for (const childIssueId of childIds) { + if ((this.record.childRetries ?? []).some(attempt => attempt.childIssueId === childIssueId && attempt.accepted)) continue; + const child = await this.api.get(`/api/issues/${childIssueId}`).catch(() => null); + const interruption = child?.interruption as Json | null | undefined; + if (interruption?.canRetry !== true) continue; + const observedRunId = typeof interruption.observedRunId === "string" ? interruption.observedRunId : null; + const attempt: RetryAttempt & { childIssueId: string } = { childIssueId, at: now, observedRunId, httpStatus: null, accepted: null, error: null }; + try { + const result = await this.api.post<{ accepted?: boolean }>(`/api/issues/${childIssueId}/retry`, { idempotencyKey: randomUUID(), observedRunId }); + attempt.httpStatus = 200; + attempt.accepted = result?.accepted === true; + attempt.acceptedAt = Math.max(now, Date.now()); + } catch (error) { + attempt.httpStatus = error instanceof ChaosApiError ? error.status : null; + attempt.error = error instanceof Error ? error.message : String(error); + } + (this.record.childRetries ??= []).push(attempt); + this.timeline.add("task_child_retry_clicked", { taskId: this.plan.taskId, ...attempt }); + clicked = clicked || attempt.accepted === true; + } + return clicked; + } + + private async collectStallWakes(issueId: string): Promise> { + const rows = await this.api.get(`/api/issues/${issueId}/activity`).catch(() => []); + return (rows ?? []).filter(row => row.action === "issue.blocker_stalled_wake").map(row => { + const details = (row.details ?? {}) as Json; + return { stalledIssueId: typeof details.stalledIssueId === "string" ? details.stalledIssueId : null, + recoveryActionId: typeof details.recoveryActionId === "string" ? details.recoveryActionId : null }; + }); + } + private async retry(sample: TaskSample, now: number): Promise { const observedRunId = sample.interruption?.observedRunId ?? (this.issue?.executionBlocker?.runId as string | undefined) ?? null; const attempt: RetryAttempt = { at: now, observedRunId, httpStatus: null, accepted: null, error: null }; @@ -770,7 +831,8 @@ export class TaskObserver { agentComments: (comments ?? []).filter(comment => comment.authorAgentId).map(comment => String(comment.body ?? "")), files, commands, workspacePath, ...(this.plan.workload === "B5" ? { benchProgressObserved: this.record.benchProgressSeen?.length ?? 0 } : {}), - ...(definition.collect.childDocuments ? { children: await this.collectChildren(issueId) } : {}), + ...(definition.collect.childDocuments ? { children: await this.collectChildren(issueId), + childStalls: this.record.childStalls ?? [], stallWakes: await this.collectStallWakes(issueId) } : {}), ...(this.plan.workload === "G1" ? { approvals: await this.collectApprovals(issueId) } : {}), ...(this.plan.workload === "H1" ? await this.finalPauseCheck(issueId) : {}), ...(this.record.governance ? { governance: this.record.governance } : {}), diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index 427985f2b6c..fb02f8a617f 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -222,6 +222,8 @@ export interface OracleArtifacts { /** B5: distinct `.bench/progress` values the harness itself saw while the task was live. */ benchProgressObserved?: number; children?: Array<{ id: string; title?: string | null; status: string | null; assigneeAgentId: string | null; documents: Record }>; + childStalls?: Array<{ childIssueId: string; recoveryActionId: string | null }>; + stallWakes?: Array<{ stalledIssueId: string | null; recoveryActionId: string | null }>; /** G1: approvals linked to the task. */ approvals?: Array<{ id: string; status: string; title: string | null }>; /** G1 and H1: the board's actions and what the observer saw around them. */ @@ -288,6 +290,8 @@ export interface TaskRecord { lastFaultAt?: number; restSnapshotError?: string; priorRetries?: RetryAttempt[]; + childStalls?: Array<{ childIssueId: string; recoveryActionId: string | null }>; + childRetries?: Array; workspaceMovedTo?: string; /** Set when the pre-fault snapshot could not read the workspace; R2 then fails visibly. */ preFaultSnapshotError?: string | null; diff --git a/tests/restart-chaos/workloads.ts b/tests/restart-chaos/workloads.ts index c1c837191f0..46693c100d6 100644 --- a/tests/restart-chaos/workloads.ts +++ b/tests/restart-chaos/workloads.ts @@ -60,6 +60,20 @@ export function parseBenchRecord(text: string | null | undefined) { ? { seconds: finished - started, result: fields.result } : null; } +export function stallWakeChecks(artifacts: OracleArtifacts): OracleResult["checks"] { + const stalls = artifacts.childStalls ?? []; + if (!stalls.length) return []; + const wakes = artifacts.stallWakes ?? []; + const matching = (stall: { childIssueId: string; recoveryActionId: string | null }) => wakes.filter(wake => + wake.stalledIssueId === stall.childIssueId && (stall.recoveryActionId === null || wake.recoveryActionId === stall.recoveryActionId)); + const unwoken = stalls.filter(stall => matching(stall).length === 0).map(stall => `${stall.childIssueId}:${stall.recoveryActionId}`); + const perPair = new Map(); + for (const wake of wakes) perPair.set(`${wake.stalledIssueId}:${wake.recoveryActionId}`, (perPair.get(`${wake.stalledIssueId}:${wake.recoveryActionId}`) ?? 0) + 1); + const repeated = [...perPair.entries()].filter(([, count]) => count > 1).map(([key]) => key); + return [{ name: "parent_woken_once_per_stall", ok: !unwoken.length && !repeated.length, + detail: `stalls=${stalls.length} wakes=${wakes.length}${unwoken.length ? ` unwoken=${unwoken.join(",")}` : ""}${repeated.length ? ` repeated=${repeated.join(",")}` : ""}` }]; +} + function result(checks: OracleResult["checks"]): OracleResult { return { verdict: checks.every(check => check.ok) ? "pass" : "fail", checks }; } @@ -205,6 +219,7 @@ export const workloadDefinitions: Record = { { name: "part2_kept", ok: distinct && Boolean(agents) && part2[0]!.assigneeAgentId === agents!.self.id, detail: `assignee=${part2[0]?.assigneeAgentId ?? "none"}` }, { name: "summary_has_both_parts", ok: summary.includes(partMarker(nonce, 1)) && summary.includes(partMarker(nonce, 2)) }, + ...stallWakeChecks(artifacts), ]); }, },