From 59b1b3787045941b4444d7cf1464ea8f419e25a7 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 12 Nov 2026 12:53:40 -0600 Subject: [PATCH] test(restart-chaos): count hands-off completion from the deliverable and never click a scheduled retry Co-Authored-By: Claude Opus 5.5 --- tests/restart-chaos/README.md | 42 ++++-- tests/restart-chaos/main.ts | 2 +- tests/restart-chaos/observer.ts | 11 +- tests/restart-chaos/release-swap-finish.ts | 2 + tests/restart-chaos/release-swap-setup.ts | 4 +- tests/restart-chaos/staging-mix-retry.test.ts | 126 ++++++++++++------ tests/restart-chaos/staging-mix-retry.ts | 102 ++++++++++---- tests/restart-chaos/staging-mix-run.ts | 2 +- tests/restart-chaos/staging-mix.ts | 29 ++-- 9 files changed, 228 insertions(+), 92 deletions(-) diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index ae19f8e408e..c87edfbdab2 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -1397,15 +1397,35 @@ own continuation is what gets measured. A task that shows a Retry banner is kept is recorded per task: when it first showed, how long it stayed, and the error code behind it, even when the task then finished by itself. A banner is a defect, because the product goal is that it never appears. After the settle (`--mix-settle-minutes`, default 15, after the last fault slot ends), each task that is still not done and still -offers Retry gets one click (`staging_mix_retry_after_settle`), and the harness follows it to rest. When the release-swap -cell drives the wave, its own end-of-wave pass makes that one click instead. Both paths use -`shouldClickRetryAfterSettle` in `staging-mix-retry.ts`: never on a done task, a task waiting on a scheduled retry, or a -task that was already clicked. So no task gets two clicks. `--mix-retry-during-wave` restores the old behaviour: Retries +offers Retry gets one click (`staging_mix_retry_after_settle`). The harness follows the click to rest for up to 15 minutes, +whatever the settle setting. A task that finished by itself during the settle is watched again until it rests, so its +record is fresh before grading. When the release-swap cell drives the wave, its own end-of-wave pass makes that one +click instead, and its issue reads also update the banner record. Both paths use `shouldClickRetryAfterSettle` in +`staging-mix-retry.ts`, which never clicks: + +- a done task; +- a task with a scheduled retry (`scheduledRetry`, state `retry_scheduled` or a `nextRetryAt`); +- a task that was already clicked. + +So no task gets two clicks. `--mix-retry-during-wave` restores the old behaviour: Retries clicked during the wave, held until an overlap window as follow-ups are (`staging_mix_retry_in_overlap`). -**Results.** `staging-mix-results.json` and the first lines of `SUMMARY.md` give three numbers, in this order: -`completed hands-off / started`, `Retry banner shown: N tasks` and `Retry worked / Retry clicked`. A Retry counts as -worked when the task is done at the end. The banner list follows, then one read of +**Results.** `staging-mix-results.json` and the first lines of `SUMMARY.md` give four lines, in this order: + +1. `completed hands-off / started`. +2. `duplicated side effects: N tasks`. +3. `Retry banner shown: N tasks`. +4. `Retry worked / Retry clicked`. + +A task counts as completed hands-off when all three hold, read from fresh data at the end of the wave: + +- its status is `done`; +- every check of its deliverable passes (ledger rounds, DONE comment posted, follow-up delivered, subtasks); +- nobody clicked Retry. + +A duplicated side effect (`rounds_once`, `done_comment_once`, `completion_once`, `subtasks_once`, +`subtask_rounds_once`) is counted on the second line. It does not remove the task from the first line. A Retry counts as +worked when the clicked task meets the same completion rule. The banner list follows, then one read of `GET /api/companies/:id/assigned-work-invariant` at the end of the wave: `violationCount`, `violations` by reason, `agentsWithStalledSlots` and `openTasks`. @@ -1417,11 +1437,13 @@ not the status: | `issue_done` | the task is not `done` | | `ledger_rounds_verified` | a round is missing, ran under 80 s, or its hash differs from the harness's own computation (POSIX `cksum`, chained per tick) | | `rounds_once` | a round line appears twice (a tool ran again) | -| `done_comment_once` | the comment whose whole body is `DONE-` is missing or was posted twice; a summary that quotes the marker does not count | -| `completion_once` | more than one run posted a comment that mentions `DONE-` (a duplicate completion) | +| `done_comment_posted` | no comment's whole body is `DONE-`; a summary that quotes the marker does not count | +| `done_comment_once` | that comment was posted more than once | +| `completion_once` | more than one run posted a comment that mentions `DONE-` (a duplicate completion), or the comments could not be read | | `follow_up_posted` | a planned follow-up was never posted (the detail lists the post errors) | | `follow_up_served` | a follow-up was posted during a run and the ledger lacks its line | -| `subtasks_once`, `subtask__done`, `parent_read_subtask_` | a parent created more or fewer than two subtasks, a subtask's ledger fails, or the parent did not report a subtask's last hash | +| `subtask__done`, `parent_read_subtask_` | a subtask is missing, not done or its ledger fails, or the parent did not report its last hash | +| `subtasks_once`, `subtask_rounds_once` | a parent created more than two subtasks, or a subtask repeated a round | Duplicated side effects (a repeated round, comment or subtask) fail the task and show in the signature. diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index 48192d8c4a9..f32e24da169 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -836,7 +836,7 @@ async function main(): Promise { rules: Object.fromEntries(Object.entries(grade.rules).map(([id, rule]) => [id, rule.verdict])), T: grade.T.verdict, universal: Object.fromEntries(Object.entries(grade.universal ?? {}).map(([id, rule]) => [id, `${rule.verdict}${rule.enforced ? "" : " (warning)"}`])) }); } - if (mix) await writeStagingMixResults({ waveDir, api, companyId: lab.companyId, tasks: launched, passed: new Map(lines.map(line => [line.taskId, line.pass])), timeline: timeline.entries }) + if (mix) await writeStagingMixResults({ waveDir, api, companyId: lab.companyId, tasks: launched, timeline: timeline.entries }) .catch(error => { harnessError ??= new Error(`Staging-mix results failed: ${String(error)}`); }); // MS1–MS3: the cross-server invariant over the whole wave (multi-server.ts). if (faults.some(receipt => receipt.multiServer)) { diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index 17c6822cd6a..c51c2917dd4 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -698,10 +698,15 @@ export class TaskObserver { canRetry: sample.interruption?.canRetry ?? false, retried: Boolean(this.record.retry) }); } - async retryAfterSettle(sample: TaskSample, now: number): Promise { + reopen(now: number): void { this.done = false; this.record.restAt = null; this.record.timedOut = false; + this.record.deadlineAt = now + RETRY_BUDGET_MS; + } + + async retryAfterSettle(sample: TaskSample, now: number): Promise { + this.reopen(now); await this.boardRetry(sample, now); this.record.deadlineAt = now + RETRY_BUDGET_MS; } @@ -1043,7 +1048,7 @@ export class TaskObserver { const document = await this.api.get(`/api/issues/${issueId}/documents/${key}`).catch(() => null); documents[key] = typeof document?.body === "string" ? document.body : null; } - const comments = await this.api.get(`/api/issues/${issueId}/comments`).catch(() => []); + const comments = await this.api.get(`/api/issues/${issueId}/comments`).catch(() => null); const workspacePath = await this.workspacePath(); const snapshotPath = this.record.preFaultWorkspacePath ?? workspacePath; if (snapshotPath && this.record.preFaultFiles) { @@ -1101,7 +1106,7 @@ export class TaskObserver { status: String(run.status), errorCode: run.errorCode ?? null, createdAt: run.createdAt ?? null, finishedAt: run.finishedAt ?? null }))).catch(() => []), ...(this.journey ? { journey: await this.journey.collect(issueId).finally(() => this.journey!.close()) } : {}), ...(this.plan.workload === "H2" ? { handoff: await this.collectHandoff(issueId, issue) } : {}), - ...(this.plan.stagingMix ? { stagingMix: await collectStagingMixArtifacts(this.api, issueId, this.record) } : {}), + ...(this.plan.stagingMix ? { stagingMix: collectStagingMixArtifacts(this.record, comments) } : {}), }; } diff --git a/tests/restart-chaos/release-swap-finish.ts b/tests/restart-chaos/release-swap-finish.ts index 4a0d9b4bf20..c33e49bf439 100644 --- a/tests/restart-chaos/release-swap-finish.ts +++ b/tests/restart-chaos/release-swap-finish.ts @@ -31,6 +31,7 @@ export interface ReleaseSwapFinishInput { record: (kind: string, data: Record) => void; pollMs?: number; drainCapMs?: number; + onView?: (issueId: string, issue: Json | null, at: number) => void; } type Client = Pick; @@ -200,6 +201,7 @@ export async function finishReleaseSwap(input: ReleaseSwapFinishInput): Promise< const left = deadline - Date.now(); views.set(id, left <= 0 ? { status: "unreadable", error: "view deadline passed" } : await input.api.get(`/api/issues/${id}`, Math.min(10_000, left)).catch(error => ({ status: "unreadable", error: String(error) }))); + input.onView?.(id, views.get(id)!, Date.now()); } })); }; diff --git a/tests/restart-chaos/release-swap-setup.ts b/tests/restart-chaos/release-swap-setup.ts index b1c5c160558..4fb96461abd 100644 --- a/tests/restart-chaos/release-swap-setup.ts +++ b/tests/restart-chaos/release-swap-setup.ts @@ -7,6 +7,7 @@ import { ReleaseSwapRun, overlapLaunchExtensionMs, type ReleaseSwapOptions } fro import { finishReleaseSwap, type ReleaseSwapFinishInput } from "./release-swap-finish.js"; import { StormRun } from "./scheduler.js"; import { setOverlapProbe } from "./staging-mix.js"; +import { bannerSink } from "./staging-mix-retry.js"; import type { ChaosSupervisor } from "./supervisor.js"; import type { TaskObserver } from "./observer.js"; import { waitForAgentsReady, type ChaosLab } from "./seed.js"; @@ -87,7 +88,8 @@ export class ReleaseSwapWave { const graded = await finishReleaseSwap({ driver: this.driver, settleMs: options.settleMs, companyId: this.lab.companyId, api: this.api, db: this.db, board: input.board, drainCapMs: options.drainMs ?? Math.max(0, ...input.observers.map(observer => observer.definition.budgetMs)) + options.settleMs, timeline: this.timeline.entries, issueIdForTask: new Map(input.launched.flatMap(observer => observer.record.issueId ? [[observer.plan.taskId, observer.record.issueId] as const] : [])), - serverWindows: this.supervisor.serverWindows, waveDir: input.waveDir, cancelled: this.cancelled, record: (kind, data) => { this.timeline.add(kind, data); } }); + serverWindows: this.supervisor.serverWindows, waveDir: input.waveDir, cancelled: this.cancelled, record: (kind, data) => { this.timeline.add(kind, data); }, + onView: bannerSink(input.launched) }); return graded.pass; } finally { setOverlapProbe(null); diff --git a/tests/restart-chaos/staging-mix-retry.test.ts b/tests/restart-chaos/staging-mix-retry.test.ts index 35e94bcc915..59db481fb51 100644 --- a/tests/restart-chaos/staging-mix-retry.test.ts +++ b/tests/restart-chaos/staging-mix-retry.test.ts @@ -1,8 +1,9 @@ import { describe, expect, it } from "vitest"; import { parseCommand } from "./config.js"; -import { doneMarker, expectedRoundHash, gradeStagingMix, planStagingMixWave, ROUND_SECONDS, type StagingMixTask } from "./staging-mix.js"; +import { collectStagingMixArtifacts, doneMarker, expectedRoundHash, gradeStagingMix, planStagingMixWave, ROUND_SECONDS, type StagingMixTask } from "./staging-mix.js"; import { - readAssignedWorkInvariant, retryAfterSettleStage, shouldClickRetryAfterSettle, stagingMixHeadline, summarizeStagingMix, type RetryStageObserver, + bannerSink, issueView, readAssignedWorkInvariant, retryAfterSettleStage, shouldClickRetryAfterSettle, stagingMixHeadline, summarizeStagingMix, + type RetryStageObserver, } from "./staging-mix-retry.js"; import type { OracleArtifacts, TaskPlan, TaskRecord, TaskSample } from "./types.js"; @@ -30,63 +31,105 @@ describe("staging-mix Retry policy", () => { expect(planStagingMixWave("w", old).tasks.every(entry => entry.stagingMix!.retryDuringWave === true)).toBe(true); }); - it("clicks once after the settle, only on a task still offering Retry that nobody clicked", () => { - expect(shouldClickRetryAfterSettle({ status: "blocked", canRetry: true, retryWait: false }, 0)).toBe(true); - expect(shouldClickRetryAfterSettle({ status: "blocked", canRetry: true, retryWait: false }, 1)).toBe(false); - expect(shouldClickRetryAfterSettle({ status: "done", canRetry: true, retryWait: false }, 0)).toBe(false); - expect(shouldClickRetryAfterSettle({ status: "blocked", canRetry: true, retryWait: true }, 0)).toBe(false); - expect(shouldClickRetryAfterSettle({ status: "in_progress", canRetry: false, retryWait: false }, 0)).toBe(false); + it("clicks once after the settle, only on a task still offering Retry that nobody clicked and that has no retry scheduled", () => { + const view = (overrides: Partial> = {}) => ({ status: "blocked", canRetry: true, retryWait: false, retryScheduled: false, ...overrides }); + expect(shouldClickRetryAfterSettle(view(), 0)).toBe(true); + expect(shouldClickRetryAfterSettle(view(), 1)).toBe(false); + expect(shouldClickRetryAfterSettle(view({ status: "done" }), 0)).toBe(false); + expect(shouldClickRetryAfterSettle(view({ retryWait: true }), 0)).toBe(false); + expect(shouldClickRetryAfterSettle(view({ canRetry: false }), 0)).toBe(false); + expect(shouldClickRetryAfterSettle(view({ retryScheduled: true }), 0)).toBe(false); + const scheduled = issueView({ status: "in_progress", interruption: { state: "retry_scheduled", canRetry: true, nextRetryAt: "2026-11-12T20:00:00Z" } }); + expect(scheduled.retryScheduled).toBe(true); + expect(shouldClickRetryAfterSettle(scheduled, 0)).toBe(false); + expect(issueView({ status: "blocked", scheduledRetry: { scheduledRetryAt: "x" }, interruption: { state: "failed", canRetry: true } }).retryScheduled).toBe(true); + expect(issueView({ status: "blocked", interruption: { state: "failed", canRetry: true } }).retryScheduled).toBe(false); }); - it("waits out the settle hands-off, then clicks once per stuck task and follows it to rest", async () => { + const harness = () => { let clock = 0; - const issues: Record> = { - i1: { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r1" } }, - i2: { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r2" } }, - i3: { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r3" } }, - }; - const observer = (taskId: string, issueId: string, retry: TaskRecord["retry"] = null): RetryStageObserver & { clicks: number } => { + const issues: Record> = {}; + const observer = (taskId: string, issueId: string, overrides: Partial = {}, finishAfterClickMs = 0) => { const self = { - plan: { taskId, stagingMix: task() } as TaskPlan, record: { issueId, retry } as TaskRecord, done: true, clicks: 0, - async poll() { if (issues[issueId]!.status === "done") self.done = true; }, + plan: { taskId, stagingMix: task() } as TaskPlan, record: { issueId, retry: null, finalStatus: "blocked", ...overrides } as TaskRecord, done: true, + clicks: 0, reopened: 0, clickedAt: null as number | null, + async poll() { + if (issues[issueId]!.status === "done" || (self.clickedAt !== null && clock - self.clickedAt >= finishAfterClickMs)) { issues[issueId] = { status: "done" }; self.done = true; } + }, + reopen() { self.done = false; self.reopened += 1; }, async retryAfterSettle(sample: TaskSample, at: number) { - self.done = false; self.clicks += 1; + self.done = false; self.clicks += 1; self.clickedAt = at; expect(sample.interruption?.observedRunId).toBe(`r${issueId.slice(1)}`); self.record.retry = { at, accepted: true } as TaskRecord["retry"]; - issues[issueId] = { status: "done" }; }, }; - return self; + return self satisfies RetryStageObserver; }; - const healsItself = observer("t1", "i1"), stuck = observer("t2", "i2"), alreadyClicked = observer("t3", "i3", { at: 1, accepted: true } as TaskRecord["retry"]); + return { issues, observer, now: () => clock, advance: (ms: number) => { clock += ms; } }; + }; + + it("waits out the settle hands-off, refreshes a task that healed, clicks once per stuck task and follows it on the retry budget", async () => { + const h = harness(); + const failed = (id: string) => ({ status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: `r${id}` } }); + Object.assign(h.issues, { i1: failed("1"), i2: failed("2"), i3: failed("3"), i4: { status: "in_progress", interruption: { state: "retry_scheduled", canRetry: true } } }); + const healsItself = h.observer("t1", "i1"), stuck = h.observer("t2", "i2", {}, 10 * 60_000); + const alreadyClicked = h.observer("t3", "i3", { retry: { at: 1, accepted: true } as TaskRecord["retry"] }), scheduled = h.observer("t4", "i4"); const notes: string[] = []; - const result = await retryAfterSettleStage({ observers: [healsItself, stuck, alreadyClicked], api: { get: async (path: string) => issues[path.split("/").pop()!] as T }, - settleUntil: 120_000, settleMs: 60_000, cancelled: () => false, note: kind => notes.push(kind), pollMs: 60_000, now: () => clock, - sleep: async ms => { clock += ms; if (clock >= 60_000) issues.i1 = { status: "done" }; } }); - expect(clock).toBeGreaterThanOrEqual(120_000); - expect(result.clicked).toEqual(["t2"]); - expect([healsItself.clicks, stuck.clicks, alreadyClicked.clicks]).toEqual([0, 1, 0]); - expect(stuck.done).toBe(true); + const result = await retryAfterSettleStage({ observers: [healsItself, stuck, alreadyClicked, scheduled], + api: { get: async (path: string) => h.issues[path.split("/").pop()!] as T }, settleUntil: 120_000, cancelled: () => false, + note: kind => notes.push(kind), pollMs: 60_000, now: h.now, sleep: async ms => { h.advance(ms); if (h.now() >= 60_000) h.issues.i1 = { status: "done" }; } }); + expect(result).toEqual({ clicked: ["t2"], refreshed: ["t1"] }); + expect([healsItself.clicks, stuck.clicks, alreadyClicked.clicks, scheduled.clicks]).toEqual([0, 1, 0, 0]); + expect(healsItself.reopened).toBe(1); + expect([healsItself.done, stuck.done]).toEqual([true, true]); expect(healsItself.record.stagingMix!.retryBanner).toMatchObject({ firstShownAt: 0, shownMs: 60_000 }); expect(stuck.record.stagingMix!.retryAfterSettle).toMatchObject({ accepted: true, issueStatusAtClick: "blocked" }); - expect(notes).toEqual(["staging_mix_settling", "staging_mix_retry_after_settle"]); + expect(notes).toEqual(["staging_mix_settling", "staging_mix_retry_after_settle", "staging_mix_refreshed_after_settle"]); }); - it("reports the three numbers first, counting the swap grader's own clicks and banners that healed by themselves", () => { + it("follows a click on the retry budget even with a zero settle", async () => { + const h = harness(); + h.issues.i2 = { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r2" } }; + const slow = h.observer("t2", "i2", {}, 12 * 60_000); + await retryAfterSettleStage({ observers: [slow], api: { get: async (path: string) => h.issues[path.split("/").pop()!] as T }, settleUntil: 0, + cancelled: () => false, note: () => undefined, now: h.now, sleep: async ms => { h.advance(ms); } }); + expect(slow.clicks).toBe(1); + expect(slow.done).toBe(true); + expect(h.now()).toBeGreaterThanOrEqual(12 * 60_000); + }); + + it("feeds the swap grader's reads into banner tracking", () => { + const record = { issueId: "i1" } as TaskRecord; + const other = { issueId: "i9" } as TaskRecord; + const sink = bannerSink([{ plan: { stagingMix: task() } as TaskPlan, record }, { plan: {} as TaskPlan, record: other }]); + sink("i1", { status: "blocked", interruption: { canRetry: true } }, 1_000); + sink("i1", { status: "unreadable" }, 2_000); + sink("i1", { status: "in_progress", interruption: null }, 31_000); + sink("i9", { status: "blocked", interruption: { canRetry: true } }, 1_000); + expect(record.stagingMix!.retryBanner).toMatchObject({ firstShownAt: 1_000, shownMs: 30_000, visibleSince: null }); + expect(other.stagingMix).toBeUndefined(); + }); + + it("counts done tasks with a verified deliverable and no click, and reports duplicates on their own line", () => { const record = (overrides: Partial = {}) => ({ issueId: null, retry: null, ...overrides }) as TaskRecord; const banner = { firstShownAt: 0, lastShownAt: 30_000, shownMs: 30_000, visibleSince: null, errorCode: "controller_lease_lost" }; + const ok = (name: string) => ({ name, ok: true }), bad = (name: string) => ({ name, ok: false }); + const clean = [ok("issue_done"), ok("ledger_rounds_verified"), ok("done_comment_posted"), ok("rounds_once"), ok("completion_once")]; const result = summarizeStagingMix([ - { taskId: "t1", issueId: "i1", record: record(), pass: true, finalStatus: "done" }, - { taskId: "t2", issueId: "i2", record: record({ stagingMix: { retryBanner: banner } }), pass: true, finalStatus: "done" }, - { taskId: "t3", issueId: "i3", record: record({ retry: { at: 1, accepted: true } as TaskRecord["retry"], stagingMix: { retryBanner: banner } }), pass: false, finalStatus: "done" }, - { taskId: "t4", issueId: "i4", record: record({ stagingMix: { retryBanner: banner } }), pass: false, finalStatus: "blocked" }, - { taskId: "t5", issueId: "i5", record: record(), pass: false, finalStatus: "in_progress" }, - ], new Map([["i4", true]]), { violationCount: 2, violations: { nothing_pending: 2 }, agentsWithStalledSlots: 1, openTasks: 3 }, 60_000); + { taskId: "t1", issueId: "i1", record: record({ finalStatus: "blocked", timedOut: true }), finalStatus: "done", checks: clean }, + { taskId: "t2", issueId: "i2", record: record({ stagingMix: { retryBanner: banner } }), finalStatus: "done", checks: [...clean.slice(0, 4), bad("completion_once")] }, + { taskId: "t3", issueId: "i3", record: record({ retry: { at: 1, accepted: true } as TaskRecord["retry"], stagingMix: { retryBanner: banner } }), finalStatus: "done", checks: clean }, + { taskId: "t4", issueId: "i4", record: record({ stagingMix: { retryBanner: banner } }), finalStatus: "blocked", checks: [bad("issue_done")] }, + { taskId: "t5", issueId: "i5", record: record(), finalStatus: "done", checks: [ok("issue_done"), bad("ledger_rounds_verified")] }, + { taskId: "t6", issueId: "i6", record: record(), finalStatus: "done", checks: null }, + ], new Set(["i4"]), { violationCount: 2, violations: { nothing_pending: 2 }, agentsWithStalledSlots: 1, openTasks: 3 }, 60_000); expect(stagingMixHeadline(result)).toEqual([ - "completed hands-off / started: 2 / 5", + "completed hands-off / started: 2 / 6", + "duplicated side effects: 1 tasks", "Retry banner shown: 3 tasks", "Retry worked / Retry clicked: 1 / 2", ]); + expect(result.duplicatedSideEffects).toEqual([{ taskId: "t2", checks: ["completion_once"] }]); expect(result.bannerShown.find(entry => entry.taskId === "t2")).toMatchObject({ completedAnyway: true, errorCode: "controller_lease_lost", shownMs: 30_000 }); expect(result.invariant.violationCount).toBe(2); }); @@ -103,7 +146,7 @@ describe("staging-mix completion comments", () => { const nonce = "b3d70e77"; const base = (comments: Array<{ body: string; runId: string | null }>): OracleArtifacts => ({ issueStatus: "done", documents: { ledger: ledger(nonce, 4) }, agentComments: comments.map(comment => comment.body), files: {}, commands: {}, workspacePath: null, - stagingMix: { followUp: null, followUpErrors: [], retries: [], comments } }); + stagingMix: collectStagingMixArtifacts({} as TaskRecord, comments.map(comment => ({ body: comment.body, authorAgentId: "a", createdByRunId: comment.runId }))) }); const check = (artifacts: OracleArtifacts, name: string) => gradeStagingMix(nonce, task(), artifacts).checks.find(entry => entry.name === name)!; it("counts only the DONE comment itself, not a closing summary that quotes it", () => { @@ -116,6 +159,13 @@ describe("staging-mix completion comments", () => { expect(check(base(comments), "completion_once").ok).toBe(true); expect(gradeStagingMix(nonce, task(), base(comments)).verdict).toBe("pass"); expect(check(base([...comments, { body: ` \`${doneMarker(nonce)}\`. `, runId: "run-c" }]), "done_comment_once")).toMatchObject({ ok: false, detail: "comments=2" }); + expect(check(base(comments.slice(0, 2)), "done_comment_posted")).toMatchObject({ ok: false, detail: "comments=0" }); + }); + + it("marks completion_once unreadable instead of passing when the comments read failed", () => { + const unread: OracleArtifacts = { ...base([{ body: doneMarker(nonce), runId: "run-b" }]), stagingMix: collectStagingMixArtifacts({} as TaskRecord, null) }; + expect(check(unread, "completion_once")).toMatchObject({ ok: false, detail: "unreadable: the task's comments could not be read" }); + expect(check(unread, "done_comment_posted").ok).toBe(true); }); it("flags a completion summary repeated by a different run as a duplicate completion", () => { diff --git a/tests/restart-chaos/staging-mix-retry.ts b/tests/restart-chaos/staging-mix-retry.ts index ba6aa006ab7..9b753c28b5e 100644 --- a/tests/restart-chaos/staging-mix-retry.ts +++ b/tests/restart-chaos/staging-mix-retry.ts @@ -1,16 +1,25 @@ -import { writeFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import type { ChaosApi } from "./api.js"; -import { bannerShownMs, noteRetryBanner } from "./staging-mix.js"; -import type { InterruptionSample, TaskPlan, TaskRecord, TaskSample } from "./types.js"; +import { bannerShownMs, DUPLICATE_CHECKS, noteRetryBanner } from "./staging-mix.js"; +import type { InterruptionSample, OracleResult, TaskPlan, TaskRecord, TaskSample } from "./types.js"; type Json = Record; -export interface IssueView { status: string; canRetry: boolean; retryWait: boolean; observedRunId: string | null; interruption: InterruptionSample | null } +export const RETRY_FOLLOW_MS = 15 * 60_000; -export function shouldClickRetryAfterSettle(view: Pick, priorClicks: number): boolean { - return view.status !== "done" && view.canRetry && !view.retryWait && priorClicks === 0; +export interface IssueView { + status: string; + canRetry: boolean; + retryWait: boolean; + retryScheduled: boolean; + observedRunId: string | null; + interruption: InterruptionSample | null; +} + +export function shouldClickRetryAfterSettle(view: Pick, priorClicks: number): boolean { + return view.status !== "done" && view.canRetry && !view.retryWait && !view.retryScheduled && priorClicks === 0; } export function issueView(issue: Json | null): IssueView { @@ -19,6 +28,7 @@ export function issueView(issue: Json | null): IssueView { status: String(issue?.status ?? "unreadable"), canRetry: interruption?.canRetry === true, retryWait: Boolean(interruption?.retryWait), + retryScheduled: Boolean(issue?.scheduledRetry) || interruption?.state === "retry_scheduled" || Boolean(interruption?.nextRetryAt), observedRunId: typeof interruption?.observedRunId === "string" ? interruption.observedRunId : null, interruption: interruption ? { state: String(interruption.state), canRetry: interruption.canRetry === true, retryWait: Boolean(interruption.retryWait), workspaceFailure: null, observedRunId: typeof interruption.observedRunId === "string" ? interruption.observedRunId : null, @@ -26,11 +36,26 @@ export function issueView(issue: Json | null): IssueView { }; } +export function noteBannerFromIssue(record: TaskRecord, issue: Json | null, at: number): void { + if (!issue || issue.status === "unreadable") return; + const view = issueView(issue); + noteRetryBanner(record.stagingMix ??= {}, view.status !== "done" && view.canRetry, at, null); +} + +export function bannerSink(observers: ReadonlyArray<{ plan: TaskPlan; record: TaskRecord }>): (issueId: string, issue: Json | null, at: number) => void { + const byIssue = new Map(observers.flatMap(observer => observer.plan.stagingMix && observer.record.issueId ? [[observer.record.issueId, observer] as const] : [])); + return (issueId, issue, at) => { + const observer = byIssue.get(issueId); + if (observer) noteBannerFromIssue(observer.record, issue, at); + }; +} + export interface RetryStageObserver { plan: TaskPlan; record: TaskRecord; done: boolean; poll(now?: number): Promise; + reopen(now: number): void; retryAfterSettle(sample: TaskSample, now: number): Promise; } @@ -38,15 +63,15 @@ export interface RetryStageInput { observers: readonly RetryStageObserver[]; api: Pick; settleUntil: number; - settleMs: number; cancelled: () => boolean; note: (kind: string, data: Record) => void; + followMs?: number; pollMs?: number; now?: () => number; sleep?: (ms: number) => Promise; } -export async function retryAfterSettleStage(input: RetryStageInput): Promise<{ clicked: string[] }> { +export async function retryAfterSettleStage(input: RetryStageInput): Promise<{ clicked: string[]; refreshed: string[] }> { const now = input.now ?? Date.now; const sleep = input.sleep ?? (ms => delay(ms)); const pollMs = input.pollMs ?? 60_000; @@ -55,9 +80,8 @@ export async function retryAfterSettleStage(input: RetryStageInput): Promise<{ c const read = async (observers: readonly RetryStageObserver[]) => { for (const observer of observers) { const issue = await input.api.get(`/api/issues/${observer.record.issueId}`, 10_000).catch(() => null); - const view = issueView(issue); - views.set(observer, view); - noteRetryBanner(observer.record.stagingMix ??= {}, view.status !== "done" && view.canRetry, now(), null); + views.set(observer, issueView(issue)); + noteBannerFromIssue(observer.record, issue, now()); } }; input.note("staging_mix_settling", { settleUntil: new Date(input.settleUntil).toISOString(), tasks: tracked.length }); @@ -66,46 +90,66 @@ export async function retryAfterSettleStage(input: RetryStageInput): Promise<{ c if (input.cancelled() || now() >= input.settleUntil || tracked.every(observer => views.get(observer)?.status === "done")) break; await sleep(Math.min(pollMs, Math.max(1, input.settleUntil - now()))); } - const clicked: RetryStageObserver[] = []; + const followed: RetryStageObserver[] = []; + const clicked: string[] = [], refreshed: string[] = []; for (const observer of tracked) { const view = views.get(observer); - if (!view || input.cancelled() || !shouldClickRetryAfterSettle(view, observer.record.retry ? 1 : 0)) continue; + if (!view || input.cancelled()) continue; + if (view.status === "done" && observer.record.finalStatus !== "done") { + observer.reopen(now()); + refreshed.push(observer.plan.taskId); + followed.push(observer); + continue; + } + if (!shouldClickRetryAfterSettle(view, observer.record.retry ? 1 : 0)) continue; const at = now(); await observer.retryAfterSettle({ at, reachable: true, serverReady: true, issueStatus: view.status, interruption: view.interruption }, at); (observer.record.stagingMix ??= {}).retryAfterSettle = { at, accepted: observer.record.retry?.accepted ?? null, issueStatusAtClick: view.status }; input.note("staging_mix_retry_after_settle", { taskId: observer.plan.taskId, accepted: observer.record.retry?.accepted ?? null }); - clicked.push(observer); + clicked.push(observer.plan.taskId); + followed.push(observer); } - const until = now() + input.settleMs; - while (clicked.some(observer => !observer.done) && !input.cancelled() && now() < until) { - for (const observer of clicked) if (!observer.done) await observer.poll(now()); + if (refreshed.length) input.note("staging_mix_refreshed_after_settle", { taskIds: refreshed }); + const until = now() + (input.followMs ?? RETRY_FOLLOW_MS); + while (followed.some(observer => !observer.done) && !input.cancelled() && now() < until) { + for (const observer of followed) if (!observer.done) await observer.poll(now()); await sleep(Math.min(pollMs, 15_000)); } - return { clicked: clicked.map(observer => observer.plan.taskId) }; + return { clicked, refreshed }; } -export interface ResultTask { taskId: string; issueId: string | null; record: TaskRecord; pass: boolean; finalStatus: string | null } +export interface ResultTask { taskId: string; issueId: string | null; record: TaskRecord; finalStatus: string | null; checks: OracleResult["checks"] | null } export interface StagingMixResult { started: number; completedHandsOff: number; + duplicatedSideEffects: Array<{ taskId: string; checks: string[] }>; bannerShown: Array<{ taskId: string; firstShownAt: string; shownMs: number; errorCode: string | null; completedAnyway: boolean }>; retryClicked: number; retryWorked: number; invariant: { violationCount: number | null; violations: Record | null; agentsWithStalledSlots: number | null; openTasks: number | null; error?: string }; } -export function summarizeStagingMix(tasks: readonly ResultTask[], extraClicks: ReadonlyMap, invariant: StagingMixResult["invariant"], now = Date.now()): StagingMixResult { +export function deliverableVerified(checks: OracleResult["checks"] | null): boolean { + return Boolean(checks?.length) && checks!.every(check => check.ok || DUPLICATE_CHECKS.has(check.name)); +} + +export function summarizeStagingMix(tasks: readonly ResultTask[], extraClicks: ReadonlySet, invariant: StagingMixResult["invariant"], now = Date.now()): StagingMixResult { const clicked = (task: ResultTask) => Boolean(task.record.retry || task.record.priorRetries?.length || (task.issueId && extraClicks.has(task.issueId))); + const completed = (task: ResultTask) => task.finalStatus === "done" && deliverableVerified(task.checks); const clickedTasks = tasks.filter(clicked); return { started: tasks.length, - completedHandsOff: tasks.filter(task => task.pass && !clicked(task)).length, + completedHandsOff: tasks.filter(task => completed(task) && !clicked(task)).length, + duplicatedSideEffects: tasks.flatMap(task => { + const failing = (task.checks ?? []).filter(check => !check.ok && DUPLICATE_CHECKS.has(check.name)).map(check => check.name); + return failing.length ? [{ taskId: task.taskId, checks: failing }] : []; + }), bannerShown: tasks.flatMap(task => task.record.stagingMix?.retryBanner ? [{ taskId: task.taskId, firstShownAt: new Date(task.record.stagingMix.retryBanner.firstShownAt).toISOString(), shownMs: bannerShownMs(task.record.stagingMix.retryBanner, now), errorCode: task.record.stagingMix.retryBanner.errorCode, completedAnyway: task.finalStatus === "done" && !clicked(task) }] : []), retryClicked: clickedTasks.length, - retryWorked: clickedTasks.filter(task => task.finalStatus === "done").length, + retryWorked: clickedTasks.filter(completed).length, invariant, }; } @@ -113,6 +157,7 @@ export function summarizeStagingMix(tasks: readonly ResultTask[], extraClicks: R export function stagingMixHeadline(result: StagingMixResult): string[] { return [ `completed hands-off / started: ${result.completedHandsOff} / ${result.started}`, + `duplicated side effects: ${result.duplicatedSideEffects.length} tasks`, `Retry banner shown: ${result.bannerShown.length} tasks`, `Retry worked / Retry clicked: ${result.retryWorked} / ${result.retryClicked}`, ]; @@ -130,15 +175,15 @@ export async function readAssignedWorkInvariant(api: Pick, comp } export async function writeStagingMixResults(input: { waveDir: string; api: Pick; companyId: string; - tasks: ReadonlyArray<{ plan: TaskPlan; record: TaskRecord }>; passed: ReadonlyMap; - timeline: ReadonlyArray<{ kind: string; [key: string]: unknown }> }): Promise { - const extraClicks = new Map(input.timeline.filter(entry => entry.kind === "release_swap_retry_clicked" && typeof entry.issueId === "string") - .map(entry => [String(entry.issueId), entry.accepted === true ? true : entry.accepted === false ? false : null])); + tasks: ReadonlyArray<{ plan: TaskPlan; record: TaskRecord }>; timeline: ReadonlyArray<{ kind: string; [key: string]: unknown }> }): Promise { + const extraClicks = new Set(input.timeline.filter(entry => entry.kind === "release_swap_retry_clicked" && typeof entry.issueId === "string").map(entry => String(entry.issueId))); const tasks: ResultTask[] = []; for (const task of input.tasks) { const issue = task.record.issueId ? await input.api.get(`/api/issues/${task.record.issueId}`, 10_000).catch(() => null) : null; - tasks.push({ taskId: task.plan.taskId, issueId: task.record.issueId, record: task.record, pass: input.passed.get(task.plan.taskId) === true, - finalStatus: issue?.status ?? task.record.finalStatus }); + const grade = await readFile(path.join(input.waveDir, `${task.plan.taskId}.grade.json`), "utf8") + .then(text => JSON.parse(text) as { grade?: { T?: OracleResult } }).catch(() => null); + tasks.push({ taskId: task.plan.taskId, issueId: task.record.issueId, record: task.record, finalStatus: issue?.status ?? task.record.finalStatus, + checks: grade?.grade?.T?.checks ?? null }); } const result = summarizeStagingMix(tasks, extraClicks, await readAssignedWorkInvariant(input.api, input.companyId)); await writeFile(path.join(input.waveDir, "staging-mix-results.json"), `${JSON.stringify({ schema: "paperclip.restart-chaos.staging-mix/v1", @@ -146,6 +191,7 @@ export async function writeStagingMixResults(input: { waveDir: string; api: Pick const invariant = result.invariant.violationCount === null ? `unreadable${result.invariant.error ? ` (${result.invariant.error})` : ""}` : `${result.invariant.violationCount} violations, ${result.invariant.agentsWithStalledSlots} agents with stalled slots, ${result.invariant.openTasks} open tasks`; await writeFile(path.join(input.waveDir, "staging-mix-summary.md"), [...stagingMixHeadline(result), `Assigned-work invariant: ${invariant}`, "", + ...result.duplicatedSideEffects.map(entry => `- ${entry.taskId}: duplicated ${entry.checks.join(", ")}`), ...result.bannerShown.map(entry => `- ${entry.taskId}: Retry banner from ${entry.firstShownAt} for ${Math.round(entry.shownMs / 1000)} s` + `${entry.errorCode ? ` (${entry.errorCode})` : ""}${entry.completedAnyway ? ", completed without a click" : ""}`), ""].join("\n"), { mode: 0o600 }); return result; diff --git a/tests/restart-chaos/staging-mix-run.ts b/tests/restart-chaos/staging-mix-run.ts index 2263ec02ded..21bc79797a0 100644 --- a/tests/restart-chaos/staging-mix-run.ts +++ b/tests/restart-chaos/staging-mix-run.ts @@ -145,7 +145,7 @@ export class StagingMixRun { async retryAfterSettle(observers: readonly RetryStageObserver[], api: Pick, cancelled: () => boolean): Promise { if (this.mix.retryDuringWave) return; - await retryAfterSettleStage({ observers, api, settleUntil: this.settleUntil(), settleMs: this.mix.settleMinutes * 60_000, cancelled, note: this.note }); + await retryAfterSettleStage({ observers, api, settleUntil: this.settleUntil(), cancelled, note: this.note }); } mayLaunch(now: number, next: TaskPlan): boolean { diff --git a/tests/restart-chaos/staging-mix.ts b/tests/restart-chaos/staging-mix.ts index 7261bbc9122..98b003f06ba 100644 --- a/tests/restart-chaos/staging-mix.ts +++ b/tests/restart-chaos/staging-mix.ts @@ -63,8 +63,11 @@ export interface StagingMixArtifacts { followUpErrors: string[]; retries: NonNullable; comments?: Array<{ body: string; runId: string | null }>; + commentsUnreadable?: boolean; } +export const DUPLICATE_CHECKS: ReadonlySet = new Set(["rounds_once", "done_comment_once", "completion_once", "subtasks_once", "subtask_rounds_once"]); + export const STAGING_MIX_ARG_OPTIONS = { workload: { type: "string" }, "mix-runs": { type: "string" }, @@ -412,10 +415,12 @@ export function gradeStagingMix(nonce: string, task: StagingMixTask, artifacts: const checks: OracleResult["checks"] = [ { name: "issue_done", ok: artifacts.issueStatus === "done", detail: `status=${artifacts.issueStatus ?? "unknown"}` }, { name: "ledger_rounds_verified", ok: own.verified, detail: own.detail }, + { name: "done_comment_posted", ok: doneComments >= 1, detail: `comments=${doneComments}` }, { name: "rounds_once", ok: own.duplicates === 0, detail: `duplicates=${own.duplicates}` }, - { name: "done_comment_once", ok: doneComments === 1, detail: `comments=${doneComments}` }, - { name: "completion_once", ok: completionRuns.length <= 1, - detail: completionRuns.length > 1 ? `duplicate completion: runs ${completionRuns.map(id => id.slice(0, 8)).join(", ")} each reported completion` : `runs=${completionRuns.length}` }, + { name: "done_comment_once", ok: doneComments <= 1, detail: `comments=${doneComments}` }, + artifacts.stagingMix?.commentsUnreadable ? { name: "completion_once", ok: false, detail: "unreadable: the task's comments could not be read" } + : { name: "completion_once", ok: completionRuns.length <= 1, + detail: completionRuns.length > 1 ? `duplicate completion: runs ${completionRuns.map(id => id.slice(0, 8)).join(", ")} each reported completion` : `runs=${completionRuns.length}` }, ]; if (task.followUpAfterMs !== null) { const errors = artifacts.stagingMix?.followUpErrors ?? []; @@ -427,16 +432,20 @@ export function gradeStagingMix(nonce: string, task: StagingMixTask, artifacts: detail: `posted ${new Date(followUp.at).toISOString()}${followUp.inOverlap ? " in overlap" : ""}` }); } if (task.subtasks > 0) { - checks.push({ name: "subtasks_once", ok: children.length === task.subtasks, detail: `subtasks=${children.length}/${task.subtasks}` }); + checks.push({ name: "subtasks_once", ok: children.length <= task.subtasks, detail: `subtasks=${children.length}/${task.subtasks}` }); + let childDuplicates = 0; for (let part = 1; part <= task.subtasks; part += 1) { const nonceOf = childNonce(nonce, part); const child = children.filter(candidate => (candidate.title ?? "").includes(nonceOf)); - const result = verifyLedger(nonceOf, SUBTASK_ROUNDS, child[0]?.documents.ledger); + const results = child.map(candidate => ({ candidate, result: verifyLedger(nonceOf, SUBTASK_ROUNDS, candidate.documents.ledger) })); + const best = results.find(entry => entry.candidate.status === "done" && entry.result.verified) ?? results[0]; + childDuplicates += results.reduce((sum, entry) => sum + entry.result.duplicates, 0); const lastHash = expectedRoundHash(nonceOf, SUBTASK_ROUNDS); - checks.push({ name: `subtask_${part}_done`, ok: child.length === 1 && child[0]!.status === "done" && result.verified && result.duplicates === 0, - detail: child.length ? `status=${child[0]!.status} ${result.detail}` : "missing" }); + checks.push({ name: `subtask_${part}_done`, ok: Boolean(best && best.candidate.status === "done" && best.result.verified), + detail: best ? `status=${best.candidate.status} ${best.result.detail}` : "missing" }); checks.push({ name: `parent_read_subtask_${part}`, ok: (ledger ?? "").includes(childLine(part, lastHash)), detail: childLine(part, lastHash) }); } + checks.push({ name: "subtask_rounds_once", ok: childDuplicates === 0, detail: `duplicates=${childDuplicates}` }); } return { verdict: checks.every(check => check.ok) ? "pass" : "fail", checks }; } @@ -561,8 +570,8 @@ export function bannerShownMs(banner: RetryBanner, now: number): number { return banner.shownMs + (banner.visibleSince === null ? 0 : now - banner.visibleSince); } -export async function collectStagingMixArtifacts(api: Pick, issueId: string, record: TaskRecord): Promise { - const rows = await api.get>(`/api/issues/${issueId}/comments`).catch(() => null); +export function collectStagingMixArtifacts(record: TaskRecord, rows: ReadonlyArray<{ body?: unknown; authorAgentId?: unknown; createdByRunId?: unknown }> | null): StagingMixArtifacts { return { followUp: record.stagingMix?.followUp ?? null, followUpErrors: record.stagingMix?.followUpErrors ?? [], retries: record.stagingMix?.retries ?? [], - ...(rows ? { comments: rows.filter(row => row.authorAgentId).map(row => ({ body: String(row.body ?? ""), runId: row.createdByRunId ?? null })) } : {}) }; + ...(rows ? { comments: rows.filter(row => row.authorAgentId).map(row => ({ body: String(row.body ?? ""), runId: typeof row.createdByRunId === "string" ? row.createdByRunId : null })) } + : { commentsUnreadable: true }) }; }