diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index a40d71aa69c..ae19f8e408e 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -1338,6 +1338,10 @@ node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona --daytona-image $IMG --daytona-warm --daytona-cpu 1 \ --storage s3 --ephemeral-server --workload staging-mix --faults K15 --k15-overlap-seconds 480-540 --mix-runs 15 --mix-queue 3 \ --mix-minutes 40 --mix-fault-spacing-minutes 12 --mix-turn-minutes 5-15 --budget-cents --seed 7102 +# The same short form locally (agents on this Mac, so 10 runs). `--ephemeral-server` runs only on Daytona, so it is left out. +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env local --storage s3 --workload staging-mix --faults K15 \ + --k15-overlap-seconds 480-540 --mix-runs 10 --mix-queue 3 --mix-minutes 40 --mix-fault-spacing-minutes 12 --mix-turn-minutes 5-15 \ + --budget-cents 5000 --seed 7201 # Full form (about 4.5 hours): 6 swaps 40 minutes apart, 15-45 minute turns, 30 runs and a queue of 6. RESTART_CHAOS_DAYTONA_VCPU_BUDGET= node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona \ --daytona-image $IMG --daytona-warm --daytona-cpu 1 --storage s3 --ephemeral-server --workload staging-mix --faults K15 \ @@ -1381,13 +1385,29 @@ one settled and at least one task is in flight; it hits every task in flight (`s tasks do: the wave's clock crosses them. A slot that never fired because every task rested first fails the wave as a harness error (`staging_mix_fault_slots_missed`). With `--faults K0` the tasks are `K0` and nothing fires. -**Follow-ups and Retries.** About half of the tasks (`--mix-follow-up-share`) get a board comment while a run is -running. The comment asks the agent to add `FOLLOWUP-` to the ledger. A failed task's Retry is clicked the -same way. Both are held until the next overlap window, but for at most `--mix-hold-minutes` (default 10); a follow-up -also goes out before the last 15% of the planned turn. A failed post is tried again on the next poll. The -overlap window comes from `setOverlapProbe` in `staging-mix.ts`, which a swap driver registers. Without a probe, the -window is any time a fault is in progress and the server still answers (a K15 overlap). The timeline records -`staging_mix_follow_up` and `staging_mix_retry_in_overlap`. +**Follow-ups.** About half of the tasks (`--mix-follow-up-share`) get a board comment while a run is running. The +comment asks the agent to add `FOLLOWUP-` to the ledger. It is held until the next overlap window, but for at +most `--mix-hold-minutes` (default 10), and it goes out before the last 15% of the planned turn. A failed post is +tried again on the next poll with the same `clientRequestId`. The overlap window comes from `setOverlapProbe` in +`staging-mix.ts`, which a swap driver registers. Without a probe, the window is any time a fault is in progress and the +server still answers (a K15 overlap). The timeline records `staging_mix_follow_up`. + +**Retries: hands-off first.** By default the harness never clicks Retry during the wave or the settle, so the product's +own continuation is what gets measured. A task that shows a Retry banner is kept under watch, not rested. Every banner +is recorded per task: when it first showed, how long it stayed, and the error code behind it, even when the task then +finished by itself. A banner is a defect, because the product goal is that it never appears. After the settle +(`--mix-settle-minutes`, default 15, after the last fault slot ends), each task that is still not done and still +offers Retry gets one click (`staging_mix_retry_after_settle`), and the harness follows it to rest. When the release-swap +cell drives the wave, its own end-of-wave pass makes that one click instead. Both paths use +`shouldClickRetryAfterSettle` in `staging-mix-retry.ts`: never on a done task, a task waiting on a scheduled retry, or a +task that was already clicked. So no task gets two clicks. `--mix-retry-during-wave` restores the old behaviour: Retries +clicked during the wave, held until an overlap window as follow-ups are (`staging_mix_retry_in_overlap`). + +**Results.** `staging-mix-results.json` and the first lines of `SUMMARY.md` give three numbers, in this order: +`completed hands-off / started`, `Retry banner shown: N tasks` and `Retry worked / Retry clicked`. A Retry counts as +worked when the task is done at the end. The banner list follows, then one read of +`GET /api/companies/:id/assigned-work-invariant` at the end of the wave: `violationCount`, `violations` by reason, +`agentsWithStalledSlots` and `openTasks`. **Grade.** The usual R1–R6, U1–U5 and T rules apply. A needed Retry still fails R3. The T-oracle checks the result, not the status: @@ -1397,7 +1417,8 @@ not the status: | `issue_done` | the task is not `done` | | `ledger_rounds_verified` | a round is missing, ran under 80 s, or its hash differs from the harness's own computation (POSIX `cksum`, chained per tick) | | `rounds_once` | a round line appears twice (a tool ran again) | -| `done_comment_once` | the `DONE-` comment is missing or was posted twice | +| `done_comment_once` | the comment whose whole body is `DONE-` is missing or was posted twice; a summary that quotes the marker does not count | +| `completion_once` | more than one run posted a comment that mentions `DONE-` (a duplicate completion) | | `follow_up_posted` | a planned follow-up was never posted (the detail lists the post errors) | | `follow_up_served` | a follow-up was posted during a run and the ledger lacks its line | | `subtasks_once`, `subtask__done`, `parent_read_subtask_` | a parent created more or fewer than two subtasks, a subtask's ledger fails, or the parent did not report a subtask's last hash | diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index 5707d54cb2f..48192d8c4a9 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -49,6 +49,7 @@ import { chaosLifecycleOverrides, seedChaosLab } from "./seed.js"; import { ChaosSupervisor } from "./supervisor.js"; import { planStagingMixWave } from "./staging-mix.js"; import { StagingMixRun, stagingMixSeedOptions } from "./staging-mix-run.js"; +import { writeStagingMixResults } from "./staging-mix-retry.js"; import { TimeShiftedApi } from "./time-faults.js"; import type { RestartStep } from "./repeated-restarts.js"; import { isBrokenStdoutError, markTimelineEchoBroken, Timeline } from "./timeline.js"; @@ -734,13 +735,8 @@ async function main(): Promise { sandboxWatchStopped = true; await mix?.stop(); await Promise.all(faultWork); - const mixShortfall = mix && !interrupted ? mix.shortfall(launched.length) : []; - if (mixShortfall.length) { - timeline.add("staging_mix_shortfall", { problems: mixShortfall, unfired: mix!.unfiredSlots() }); - harnessError ??= new Error(`Staging mix tested less than planned: ${mixShortfall.join("; ")}`); - } - if (mix) console.info(`Staging mix: ${launched.length} tasks started; ${mix.unlaunched} planned tasks never launched (excluded from tasks started); ` - + `fault slots fired ${mix.slotCoverage.map(entry => `${Math.round(entry.atMs / 60_000)}m:${entry.kind}×${entry.exposed} tasks`).join(", ") || "none"} of ${mix.slots.length}`); + harnessError ??= mix && !interrupted ? mix.report(launched.length) ?? undefined : undefined; + if (mix && !interrupted && !releaseSwap) await mix.retryAfterSettle(launched, api, () => interrupted); if (storageTrigger) { const shortfall = storageTriggerShortfall(storageTrigger.finish(faults)); storageSection = storageTriggerMarkdown(storageTrigger.receipts); @@ -840,6 +836,8 @@ async function main(): Promise { rules: Object.fromEntries(Object.entries(grade.rules).map(([id, rule]) => [id, rule.verdict])), T: grade.T.verdict, universal: Object.fromEntries(Object.entries(grade.universal ?? {}).map(([id, rule]) => [id, `${rule.verdict}${rule.enforced ? "" : " (warning)"}`])) }); } + if (mix) await writeStagingMixResults({ waveDir, api, companyId: lab.companyId, tasks: launched, passed: new Map(lines.map(line => [line.taskId, line.pass])), timeline: timeline.entries }) + .catch(error => { harnessError ??= new Error(`Staging-mix results failed: ${String(error)}`); }); // MS1–MS3: the cross-server invariant over the whole wave (multi-server.ts). if (faults.some(receipt => receipt.multiServer)) { const crossServer = await readCrossServerEvidence(evidenceDb, lab.companyId) @@ -987,7 +985,7 @@ async function main(): Promise { if (lines.length) { await appendLedger(resultsRoot, lines); const summary = await writeSummary(resultsRoot, plan.waveId); - await writeFile(path.join(waveDir, "SUMMARY.md"), [await readFile(path.join(waveDir, "release-swap-summary.md"), "utf8").catch(() => null), summary, warmShutdownSection, speedSection, storageSection].filter(Boolean).join("\n"), { mode: 0o600 }); + await writeFile(path.join(waveDir, "SUMMARY.md"), [...await Promise.all(["staging-mix-summary.md", "release-swap-summary.md"].map(file => readFile(path.join(waveDir, file), "utf8").catch(() => null))), summary, warmShutdownSection, speedSection, storageSection].filter(Boolean).join("\n"), { mode: 0o600 }); console.info(`\n${summary}`); } } diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index bdd9242cd77..17c6822cd6a 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -20,7 +20,7 @@ import { awaitingQuestionAnswer, collectQuestionArtifacts, questionStep } from " import { isSessionFault, rotateAgentSession } from "./session-faults.js"; import { awaitingRewake, collectRewakeArtifacts, isRewakeWorkload, rewakeStep } from "./rewake.js"; import { withLongTurn } from "./long-turn.js"; -import { holdsRetry, stagingMixArtifacts, stagingMixDefinition, stagingMixStep } from "./staging-mix.js"; +import { collectStagingMixArtifacts, holdsRetry, stagingMixDefinition, stagingMixStep } from "./staging-mix.js"; import { captureRemoteWorkspaceFiles, readRemoteWorkspaceFile } from "./remote-workspace-files.js"; import { taskFaultSchedule } from "./scheduler.js"; import { g1ChildTitle, g1Decision, workloadDefinitions, type WorkloadAgents } from "./workloads.js"; @@ -686,8 +686,9 @@ export class TaskObserver { } if (!await this.clearDiskFullAtRest(now)) { this.restCandidateSince = null; return; } if (status !== "done" && (!this.record.retry || faultSinceRetry) && sample.interruption?.canRetry) { - if (this.plan.stagingMix && holdsRetry(this.record, this.plan.stagingMix.holdMs, now)) { this.restCandidateSince = null; return; } - await this.boardRetry(sample, now); + if (this.plan.stagingMix?.retryDuringWave && holdsRetry(this.record, this.plan.stagingMix.holdMs, now)) { this.restCandidateSince = null; return; } + if (!this.plan.stagingMix || this.plan.stagingMix.retryDuringWave) { await this.boardRetry(sample, now); return; } + this.restCandidateSince = null; return; } this.record.restAt = this.restCandidateSince ?? now; @@ -697,6 +698,14 @@ export class TaskObserver { canRetry: sample.interruption?.canRetry ?? false, retried: Boolean(this.record.retry) }); } + async retryAfterSettle(sample: TaskSample, now: number): Promise { + this.done = false; + this.record.restAt = null; + this.record.timedOut = false; + await this.boardRetry(sample, now); + this.record.deadlineAt = now + RETRY_BUDGET_MS; + } + private async boardRetry(sample: TaskSample, now: number): Promise { if (this.record.retry) (this.record.priorRetries ??= []).push(this.record.retry); this.record.retry = await this.retry(sample, now); @@ -1092,7 +1101,7 @@ export class TaskObserver { status: String(run.status), errorCode: run.errorCode ?? null, createdAt: run.createdAt ?? null, finishedAt: run.finishedAt ?? null }))).catch(() => []), ...(this.journey ? { journey: await this.journey.collect(issueId).finally(() => this.journey!.close()) } : {}), ...(this.plan.workload === "H2" ? { handoff: await this.collectHandoff(issueId, issue) } : {}), - ...(this.plan.stagingMix ? { stagingMix: stagingMixArtifacts(this.record) } : {}), + ...(this.plan.stagingMix ? { stagingMix: await collectStagingMixArtifacts(this.api, issueId, this.record) } : {}), }; } diff --git a/tests/restart-chaos/release-swap-finish.ts b/tests/restart-chaos/release-swap-finish.ts index 6a6829db7de..4a0d9b4bf20 100644 --- a/tests/restart-chaos/release-swap-finish.ts +++ b/tests/restart-chaos/release-swap-finish.ts @@ -9,6 +9,7 @@ import { gradeReleaseSwapTask, overlappingRuns, releaseSwapMarkdown, summarizeGrades, swapMetrics, type DuplicateFacts, type GradeTaskInput, type RetryClick, type SwapRunFact, type TaskGrade, } from "./release-swap-grade.js"; +import { issueView, shouldClickRetryAfterSettle } from "./staging-mix-retry.js"; import type { TimelineEntry } from "./timeline.js"; import type { Db } from "../../packages/db/src/index.js"; @@ -208,7 +209,7 @@ export async function finishReleaseSwap(input: ReleaseSwapFinishInput): Promise< for (const issue of issues) { const id = String(issue.id); const current = views.get(id)!; - if (current.status === "done" || current.interruption?.canRetry !== true || current.interruption?.retryWait) continue; + if (!shouldClickRetryAfterSettle(issueView(current), clicks.get(id)?.length ?? 0)) continue; const click = await clickRetry(input, id, text(issue.identifier) ?? id, text(current.interruption?.observedRunId)); clicks.set(id, [...(clicks.get(id) ?? []), click]); retried.push(id); diff --git a/tests/restart-chaos/staging-mix-retry.test.ts b/tests/restart-chaos/staging-mix-retry.test.ts new file mode 100644 index 00000000000..35e94bcc915 --- /dev/null +++ b/tests/restart-chaos/staging-mix-retry.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from "vitest"; +import { parseCommand } from "./config.js"; +import { doneMarker, expectedRoundHash, gradeStagingMix, planStagingMixWave, ROUND_SECONDS, type StagingMixTask } from "./staging-mix.js"; +import { + readAssignedWorkInvariant, retryAfterSettleStage, shouldClickRetryAfterSettle, stagingMixHeadline, summarizeStagingMix, type RetryStageObserver, +} from "./staging-mix-retry.js"; +import type { OracleArtifacts, TaskPlan, TaskRecord, TaskSample } from "./types.js"; + +const wave = (argv: string[]) => { + const command = parseCommand(argv); + if (command.mode !== "wave") throw new Error("expected a wave"); + return command.options; +}; + +const task = (overrides: Partial = {}): StagingMixTask => ({ + turnMinutes: 5, rounds: 4, subtasks: 0, followUpAfterMs: null, arriveAtMs: 0, holdMs: 600_000, ...overrides, +}); + +const ledger = (nonce: string, rounds: number) => Array.from({ length: rounds }, (_, index) => + `ROUND ${index + 1} ${1_800_000_000 + index * 100} ${1_800_000_000 + index * 100 + ROUND_SECONDS} ${expectedRoundHash(nonce, index + 1)}`).join("\n"); + +describe("staging-mix Retry policy", () => { + it("is hands-off by default, and keeps the overlap Retry only behind --mix-retry-during-wave", () => { + const base = ["--workload", "staging-mix", "--faults", "K0", "--mix-turn-minutes", "5-15", "--mix-minutes", "40"]; + const handsOff = wave(base); + expect(handsOff.stagingMix).toMatchObject({ retryDuringWave: false, settleMinutes: 15 }); + expect(planStagingMixWave("w", handsOff).tasks.every(entry => entry.stagingMix!.retryDuringWave === undefined)).toBe(true); + const old = wave([...base, "--mix-retry-during-wave", "--mix-settle-minutes", "5"]); + expect(old.stagingMix).toMatchObject({ retryDuringWave: true, settleMinutes: 5 }); + expect(planStagingMixWave("w", old).tasks.every(entry => entry.stagingMix!.retryDuringWave === true)).toBe(true); + }); + + it("clicks once after the settle, only on a task still offering Retry that nobody clicked", () => { + expect(shouldClickRetryAfterSettle({ status: "blocked", canRetry: true, retryWait: false }, 0)).toBe(true); + expect(shouldClickRetryAfterSettle({ status: "blocked", canRetry: true, retryWait: false }, 1)).toBe(false); + expect(shouldClickRetryAfterSettle({ status: "done", canRetry: true, retryWait: false }, 0)).toBe(false); + expect(shouldClickRetryAfterSettle({ status: "blocked", canRetry: true, retryWait: true }, 0)).toBe(false); + expect(shouldClickRetryAfterSettle({ status: "in_progress", canRetry: false, retryWait: false }, 0)).toBe(false); + }); + + it("waits out the settle hands-off, then clicks once per stuck task and follows it to rest", async () => { + let clock = 0; + const issues: Record> = { + i1: { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r1" } }, + i2: { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r2" } }, + i3: { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r3" } }, + }; + const observer = (taskId: string, issueId: string, retry: TaskRecord["retry"] = null): RetryStageObserver & { clicks: number } => { + const self = { + plan: { taskId, stagingMix: task() } as TaskPlan, record: { issueId, retry } as TaskRecord, done: true, clicks: 0, + async poll() { if (issues[issueId]!.status === "done") self.done = true; }, + async retryAfterSettle(sample: TaskSample, at: number) { + self.done = false; self.clicks += 1; + expect(sample.interruption?.observedRunId).toBe(`r${issueId.slice(1)}`); + self.record.retry = { at, accepted: true } as TaskRecord["retry"]; + issues[issueId] = { status: "done" }; + }, + }; + return self; + }; + const healsItself = observer("t1", "i1"), stuck = observer("t2", "i2"), alreadyClicked = observer("t3", "i3", { at: 1, accepted: true } as TaskRecord["retry"]); + const notes: string[] = []; + const result = await retryAfterSettleStage({ observers: [healsItself, stuck, alreadyClicked], api: { get: async (path: string) => issues[path.split("/").pop()!] as T }, + settleUntil: 120_000, settleMs: 60_000, cancelled: () => false, note: kind => notes.push(kind), pollMs: 60_000, now: () => clock, + sleep: async ms => { clock += ms; if (clock >= 60_000) issues.i1 = { status: "done" }; } }); + expect(clock).toBeGreaterThanOrEqual(120_000); + expect(result.clicked).toEqual(["t2"]); + expect([healsItself.clicks, stuck.clicks, alreadyClicked.clicks]).toEqual([0, 1, 0]); + expect(stuck.done).toBe(true); + expect(healsItself.record.stagingMix!.retryBanner).toMatchObject({ firstShownAt: 0, shownMs: 60_000 }); + expect(stuck.record.stagingMix!.retryAfterSettle).toMatchObject({ accepted: true, issueStatusAtClick: "blocked" }); + expect(notes).toEqual(["staging_mix_settling", "staging_mix_retry_after_settle"]); + }); + + it("reports the three numbers first, counting the swap grader's own clicks and banners that healed by themselves", () => { + const record = (overrides: Partial = {}) => ({ issueId: null, retry: null, ...overrides }) as TaskRecord; + const banner = { firstShownAt: 0, lastShownAt: 30_000, shownMs: 30_000, visibleSince: null, errorCode: "controller_lease_lost" }; + const result = summarizeStagingMix([ + { taskId: "t1", issueId: "i1", record: record(), pass: true, finalStatus: "done" }, + { taskId: "t2", issueId: "i2", record: record({ stagingMix: { retryBanner: banner } }), pass: true, finalStatus: "done" }, + { taskId: "t3", issueId: "i3", record: record({ retry: { at: 1, accepted: true } as TaskRecord["retry"], stagingMix: { retryBanner: banner } }), pass: false, finalStatus: "done" }, + { taskId: "t4", issueId: "i4", record: record({ stagingMix: { retryBanner: banner } }), pass: false, finalStatus: "blocked" }, + { taskId: "t5", issueId: "i5", record: record(), pass: false, finalStatus: "in_progress" }, + ], new Map([["i4", true]]), { violationCount: 2, violations: { nothing_pending: 2 }, agentsWithStalledSlots: 1, openTasks: 3 }, 60_000); + expect(stagingMixHeadline(result)).toEqual([ + "completed hands-off / started: 2 / 5", + "Retry banner shown: 3 tasks", + "Retry worked / Retry clicked: 1 / 2", + ]); + expect(result.bannerShown.find(entry => entry.taskId === "t2")).toMatchObject({ completedAnyway: true, errorCode: "controller_lease_lost", shownMs: 30_000 }); + expect(result.invariant.violationCount).toBe(2); + }); + + it("reads the assigned-work invariant once as counts, and records a failed read", async () => { + const report = { checked: 4, openTasks: 3, violations: { nothing_pending: 1 }, violationCount: 1, agentsWithStalledSlots: [{ agentId: "a" }] }; + expect(await readAssignedWorkInvariant({ get: async () => report as T }, "c1")) + .toEqual({ violationCount: 1, violations: { nothing_pending: 1 }, agentsWithStalledSlots: 1, openTasks: 3 }); + expect(await readAssignedWorkInvariant({ get: async () => { throw new Error("HTTP 404"); } }, "c1")).toMatchObject({ violationCount: null, error: "Error: HTTP 404" }); + }); +}); + +describe("staging-mix completion comments", () => { + const nonce = "b3d70e77"; + const base = (comments: Array<{ body: string; runId: string | null }>): OracleArtifacts => ({ issueStatus: "done", documents: { ledger: ledger(nonce, 4) }, + agentComments: comments.map(comment => comment.body), files: {}, commands: {}, workspacePath: null, + stagingMix: { followUp: null, followUpErrors: [], retries: [], comments } }); + const check = (artifacts: OracleArtifacts, name: string) => gradeStagingMix(nonce, task(), artifacts).checks.find(entry => entry.name === name)!; + + it("counts only the DONE comment itself, not a closing summary that quotes it", () => { + const comments = [ + { body: `CHA-1 complete. Saved the ledger and posted \`${doneMarker(nonce)}\`.`, runId: "run-b" }, + { body: "Prepared CHA-1 ledger for this response.", runId: "run-b" }, + { body: doneMarker(nonce), runId: "run-b" }, + ]; + expect(check(base(comments), "done_comment_once")).toMatchObject({ ok: true, detail: "comments=1" }); + expect(check(base(comments), "completion_once").ok).toBe(true); + expect(gradeStagingMix(nonce, task(), base(comments)).verdict).toBe("pass"); + expect(check(base([...comments, { body: ` \`${doneMarker(nonce)}\`. `, runId: "run-c" }]), "done_comment_once")).toMatchObject({ ok: false, detail: "comments=2" }); + }); + + it("flags a completion summary repeated by a different run as a duplicate completion", () => { + const comments = [ + { body: `CHA-1 complete. Posted exactly one comment ${doneMarker(nonce)} and marked the task done.`, runId: "413ddcf2-ffce" }, + { body: `CHA-1 complete. Posted \`${doneMarker(nonce)}\`.`, runId: "13de31be-fb13" }, + { body: doneMarker(nonce), runId: "13de31be-fb13" }, + ]; + expect(check(base(comments), "done_comment_once").ok).toBe(true); + expect(check(base(comments), "completion_once")).toMatchObject({ ok: false, detail: "duplicate completion: runs 413ddcf2, 13de31be each reported completion" }); + }); +}); diff --git a/tests/restart-chaos/staging-mix-retry.ts b/tests/restart-chaos/staging-mix-retry.ts new file mode 100644 index 00000000000..ba6aa006ab7 --- /dev/null +++ b/tests/restart-chaos/staging-mix-retry.ts @@ -0,0 +1,152 @@ +import { writeFile } from "node:fs/promises"; +import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import type { ChaosApi } from "./api.js"; +import { bannerShownMs, noteRetryBanner } from "./staging-mix.js"; +import type { InterruptionSample, TaskPlan, TaskRecord, TaskSample } from "./types.js"; + +type Json = Record; + +export interface IssueView { status: string; canRetry: boolean; retryWait: boolean; observedRunId: string | null; interruption: InterruptionSample | null } + +export function shouldClickRetryAfterSettle(view: Pick, priorClicks: number): boolean { + return view.status !== "done" && view.canRetry && !view.retryWait && priorClicks === 0; +} + +export function issueView(issue: Json | null): IssueView { + const interruption = issue?.interruption ?? null; + return { + status: String(issue?.status ?? "unreadable"), + canRetry: interruption?.canRetry === true, + retryWait: Boolean(interruption?.retryWait), + observedRunId: typeof interruption?.observedRunId === "string" ? interruption.observedRunId : null, + interruption: interruption ? { state: String(interruption.state), canRetry: interruption.canRetry === true, retryWait: Boolean(interruption.retryWait), + workspaceFailure: null, observedRunId: typeof interruption.observedRunId === "string" ? interruption.observedRunId : null, + userAction: typeof interruption.userAction === "string" ? interruption.userAction : null } : null, + }; +} + +export interface RetryStageObserver { + plan: TaskPlan; + record: TaskRecord; + done: boolean; + poll(now?: number): Promise; + retryAfterSettle(sample: TaskSample, now: number): Promise; +} + +export interface RetryStageInput { + observers: readonly RetryStageObserver[]; + api: Pick; + settleUntil: number; + settleMs: number; + cancelled: () => boolean; + note: (kind: string, data: Record) => void; + pollMs?: number; + now?: () => number; + sleep?: (ms: number) => Promise; +} + +export async function retryAfterSettleStage(input: RetryStageInput): Promise<{ clicked: string[] }> { + const now = input.now ?? Date.now; + const sleep = input.sleep ?? (ms => delay(ms)); + const pollMs = input.pollMs ?? 60_000; + const tracked = input.observers.filter(observer => observer.record.issueId && observer.plan.stagingMix); + const views = new Map(); + const read = async (observers: readonly RetryStageObserver[]) => { + for (const observer of observers) { + const issue = await input.api.get(`/api/issues/${observer.record.issueId}`, 10_000).catch(() => null); + const view = issueView(issue); + views.set(observer, view); + noteRetryBanner(observer.record.stagingMix ??= {}, view.status !== "done" && view.canRetry, now(), null); + } + }; + input.note("staging_mix_settling", { settleUntil: new Date(input.settleUntil).toISOString(), tasks: tracked.length }); + for (;;) { + await read(tracked.filter(observer => views.get(observer)?.status !== "done")); + if (input.cancelled() || now() >= input.settleUntil || tracked.every(observer => views.get(observer)?.status === "done")) break; + await sleep(Math.min(pollMs, Math.max(1, input.settleUntil - now()))); + } + const clicked: RetryStageObserver[] = []; + for (const observer of tracked) { + const view = views.get(observer); + if (!view || input.cancelled() || !shouldClickRetryAfterSettle(view, observer.record.retry ? 1 : 0)) continue; + const at = now(); + await observer.retryAfterSettle({ at, reachable: true, serverReady: true, issueStatus: view.status, interruption: view.interruption }, at); + (observer.record.stagingMix ??= {}).retryAfterSettle = { at, accepted: observer.record.retry?.accepted ?? null, issueStatusAtClick: view.status }; + input.note("staging_mix_retry_after_settle", { taskId: observer.plan.taskId, accepted: observer.record.retry?.accepted ?? null }); + clicked.push(observer); + } + const until = now() + input.settleMs; + while (clicked.some(observer => !observer.done) && !input.cancelled() && now() < until) { + for (const observer of clicked) if (!observer.done) await observer.poll(now()); + await sleep(Math.min(pollMs, 15_000)); + } + return { clicked: clicked.map(observer => observer.plan.taskId) }; +} + +export interface ResultTask { taskId: string; issueId: string | null; record: TaskRecord; pass: boolean; finalStatus: string | null } + +export interface StagingMixResult { + started: number; + completedHandsOff: number; + bannerShown: Array<{ taskId: string; firstShownAt: string; shownMs: number; errorCode: string | null; completedAnyway: boolean }>; + retryClicked: number; + retryWorked: number; + invariant: { violationCount: number | null; violations: Record | null; agentsWithStalledSlots: number | null; openTasks: number | null; error?: string }; +} + +export function summarizeStagingMix(tasks: readonly ResultTask[], extraClicks: ReadonlyMap, invariant: StagingMixResult["invariant"], now = Date.now()): StagingMixResult { + const clicked = (task: ResultTask) => Boolean(task.record.retry || task.record.priorRetries?.length || (task.issueId && extraClicks.has(task.issueId))); + const clickedTasks = tasks.filter(clicked); + return { + started: tasks.length, + completedHandsOff: tasks.filter(task => task.pass && !clicked(task)).length, + bannerShown: tasks.flatMap(task => task.record.stagingMix?.retryBanner ? [{ taskId: task.taskId, firstShownAt: new Date(task.record.stagingMix.retryBanner.firstShownAt).toISOString(), + shownMs: bannerShownMs(task.record.stagingMix.retryBanner, now), errorCode: task.record.stagingMix.retryBanner.errorCode, + completedAnyway: task.finalStatus === "done" && !clicked(task) }] : []), + retryClicked: clickedTasks.length, + retryWorked: clickedTasks.filter(task => task.finalStatus === "done").length, + invariant, + }; +} + +export function stagingMixHeadline(result: StagingMixResult): string[] { + return [ + `completed hands-off / started: ${result.completedHandsOff} / ${result.started}`, + `Retry banner shown: ${result.bannerShown.length} tasks`, + `Retry worked / Retry clicked: ${result.retryWorked} / ${result.retryClicked}`, + ]; +} + +export async function readAssignedWorkInvariant(api: Pick, companyId: string): Promise { + try { + const report = await api.get(`/api/companies/${companyId}/assigned-work-invariant`, 30_000); + return { violationCount: typeof report.violationCount === "number" ? report.violationCount : null, violations: report.violations ?? null, + agentsWithStalledSlots: Array.isArray(report.agentsWithStalledSlots) ? report.agentsWithStalledSlots.length : null, + openTasks: typeof report.openTasks === "number" ? report.openTasks : null }; + } catch (error) { + return { violationCount: null, violations: null, agentsWithStalledSlots: null, openTasks: null, error: String(error) }; + } +} + +export async function writeStagingMixResults(input: { waveDir: string; api: Pick; companyId: string; + tasks: ReadonlyArray<{ plan: TaskPlan; record: TaskRecord }>; passed: ReadonlyMap; + timeline: ReadonlyArray<{ kind: string; [key: string]: unknown }> }): Promise { + const extraClicks = new Map(input.timeline.filter(entry => entry.kind === "release_swap_retry_clicked" && typeof entry.issueId === "string") + .map(entry => [String(entry.issueId), entry.accepted === true ? true : entry.accepted === false ? false : null])); + const tasks: ResultTask[] = []; + for (const task of input.tasks) { + const issue = task.record.issueId ? await input.api.get(`/api/issues/${task.record.issueId}`, 10_000).catch(() => null) : null; + tasks.push({ taskId: task.plan.taskId, issueId: task.record.issueId, record: task.record, pass: input.passed.get(task.plan.taskId) === true, + finalStatus: issue?.status ?? task.record.finalStatus }); + } + const result = summarizeStagingMix(tasks, extraClicks, await readAssignedWorkInvariant(input.api, input.companyId)); + await writeFile(path.join(input.waveDir, "staging-mix-results.json"), `${JSON.stringify({ schema: "paperclip.restart-chaos.staging-mix/v1", + headline: stagingMixHeadline(result), ...result }, null, 2)}\n`, { mode: 0o600 }); + const invariant = result.invariant.violationCount === null ? `unreadable${result.invariant.error ? ` (${result.invariant.error})` : ""}` + : `${result.invariant.violationCount} violations, ${result.invariant.agentsWithStalledSlots} agents with stalled slots, ${result.invariant.openTasks} open tasks`; + await writeFile(path.join(input.waveDir, "staging-mix-summary.md"), [...stagingMixHeadline(result), `Assigned-work invariant: ${invariant}`, "", + ...result.bannerShown.map(entry => `- ${entry.taskId}: Retry banner from ${entry.firstShownAt} for ${Math.round(entry.shownMs / 1000)} s` + + `${entry.errorCode ? ` (${entry.errorCode})` : ""}${entry.completedAnyway ? ", completed without a click" : ""}`), ""].join("\n"), { mode: 0o600 }); + return result; +} diff --git a/tests/restart-chaos/staging-mix-run.ts b/tests/restart-chaos/staging-mix-run.ts index befe186657c..2263ec02ded 100644 --- a/tests/restart-chaos/staging-mix-run.ts +++ b/tests/restart-chaos/staging-mix-run.ts @@ -3,7 +3,9 @@ import { readFileSync } from "node:fs"; import { writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import type { ChaosApi } from "./api.js"; import { agentRunCaps, type StagingMixOptions, type StagingMixSlot } from "./staging-mix.js"; +import { retryAfterSettleStage, type RetryStageObserver } from "./staging-mix-retry.js"; import type { AdapterId, TaskPlan, WaveOptions } from "./types.js"; export const RESOURCE_SAMPLE_MS = 30_000; @@ -128,6 +130,24 @@ export class StagingMixRun { return problems; } + report(started: number): Error | null { + const problems = this.shortfall(started); + if (problems.length) this.note("staging_mix_shortfall", { problems, unfired: this.unfiredSlots() }); + console.info(`Staging mix: ${started} tasks started; ${this.unlaunched} planned tasks never launched (excluded from tasks started); ` + + `fault slots fired ${this.slotCoverage.map(entry => `${Math.round(entry.atMs / 60_000)}m:${entry.kind}×${entry.exposed} tasks`).join(", ") || "none"} of ${this.slots.length}`); + return problems.length ? new Error(`Staging mix tested less than planned: ${problems.join("; ")}`) : null; + } + + settleUntil(now = Date.now()): number { + const last = this.slots.at(-1); + return Math.max(now, last ? this.startedAt + last.atMs + last.holdMs : now) + this.mix.settleMinutes * 60_000; + } + + async retryAfterSettle(observers: readonly RetryStageObserver[], api: Pick, cancelled: () => boolean): Promise { + if (this.mix.retryDuringWave) return; + await retryAfterSettleStage({ observers, api, settleUntil: this.settleUntil(), settleMs: this.mix.settleMinutes * 60_000, cancelled, note: this.note }); + } + mayLaunch(now: number, next: TaskPlan): boolean { if (this.ended || !next.stagingMix || now < this.startedAt + next.stagingMix.arriveAtMs) return false; const latest = this.samples[this.samples.length - 1]; diff --git a/tests/restart-chaos/staging-mix.test.ts b/tests/restart-chaos/staging-mix.test.ts index 565b15ebcc9..b2b08a8c4d7 100644 --- a/tests/restart-chaos/staging-mix.test.ts +++ b/tests/restart-chaos/staging-mix.test.ts @@ -264,6 +264,7 @@ describe("staging-mix board step", () => { const failed: TaskSample = { at: 0, reachable: true, serverReady: true, issueStatus: "blocked", interruption: { state: "failed", canRetry: true, retryWait: false, workspaceFailure: null, observedRunId: "r1", userAction: null }, runs: [{ ...running.runs![0]!, status: "failed" }] }; + const stepIn = (rec: TaskRecord, sample: TaskSample, now: number, faultInProgress: boolean, plan: StagingMixTask) => step(rec, sample, now, faultInProgress, plan); const step = async (rec: TaskRecord, sample: TaskSample, now: number, faultInProgress: boolean, plan = task({ followUpAfterMs: 60_000 }), reply: () => unknown = () => ({ id: "c1" })) => { const posted: string[] = [], retried: number[] = [], notes: string[] = []; @@ -309,8 +310,18 @@ describe("staging-mix board step", () => { expect((await step(rec, running, Math.round(4 * ROUND_SECONDS * 1_000 * 0.85), false, plan)).posted.length).toBe(1); }); - it("waits for an overlap to click a Retry, and holds the observer's own click until the hold runs out", async () => { + it("never clicks Retry during the wave by default, but records the banner", async () => { const rec = record(); + expect((await step(rec, failed, 1_000, true)).retried).toEqual([]); + expect((await step(rec, failed, 9_000, true)).retried).toEqual([]); + expect(rec.stagingMix!.retryBanner).toMatchObject({ firstShownAt: 1_000, lastShownAt: 9_000, visibleSince: 1_000 }); + await step(rec, running, 20_000, false); + expect(rec.stagingMix!.retryBanner).toMatchObject({ shownMs: 19_000, visibleSince: null }); + }); + + it("with --mix-retry-during-wave, waits for an overlap to click a Retry, and holds the observer's own click until the hold runs out", async () => { + const rec = record(); + const step = (r: TaskRecord, sample: TaskSample, now: number, overlap: boolean) => stepIn(r, sample, now, overlap, task({ followUpAfterMs: 60_000, retryDuringWave: true })); expect((await step(rec, failed, 1_000, false)).retried).toEqual([]); expect(holdsRetry(rec, 600_000, 2_000)).toBe(true); expect(holdsRetry(rec, 600_000, 1_000 + 600_000)).toBe(false); diff --git a/tests/restart-chaos/staging-mix.ts b/tests/restart-chaos/staging-mix.ts index 37028875903..7261bbc9122 100644 --- a/tests/restart-chaos/staging-mix.ts +++ b/tests/restart-chaos/staging-mix.ts @@ -30,6 +30,8 @@ export interface StagingMixOptions { minFreeMb: number; arrivalCutoffMinutes: number; arrivalEndMs: number; + retryDuringWave: boolean; + settleMinutes: number; tasks?: number; } @@ -42,6 +44,7 @@ export interface StagingMixTask { followUpAfterMs: number | null; arriveAtMs: number; holdMs: number; + retryDuringWave?: boolean; } export interface StagingMixRecord { @@ -49,12 +52,17 @@ export interface StagingMixRecord { followUp?: { at: number; commentId: string | null; duringRun: boolean; inOverlap: boolean; error: string | null }; retrySeenAt?: number | null; retries?: Array<{ at: number; inOverlap: boolean }>; + retryBanner?: RetryBanner; + retryAfterSettle?: { at: number; accepted: boolean | null; issueStatusAtClick: string | null }; } +export interface RetryBanner { firstShownAt: number; lastShownAt: number; shownMs: number; visibleSince: number | null; errorCode: string | null } + export interface StagingMixArtifacts { followUp: StagingMixRecord["followUp"] | null; followUpErrors: string[]; retries: NonNullable; + comments?: Array<{ body: string; runId: string | null }>; } export const STAGING_MIX_ARG_OPTIONS = { @@ -71,6 +79,8 @@ export const STAGING_MIX_ARG_OPTIONS = { "mix-run-cap": { type: "string" }, "mix-tasks": { type: "string" }, "mix-arrival-cutoff-minutes": { type: "string" }, + "mix-retry-during-wave": { type: "boolean" }, + "mix-settle-minutes": { type: "string" }, } as const; type MixArgs = Partial> & Record; @@ -141,6 +151,8 @@ export function applyStagingMix(options: WaveOptions, values: MixArgs, environme minFreeMb: whole(environment.RESTART_CHAOS_MIX_MIN_FREE_MB, "RESTART_CHAOS_MIX_MIN_FREE_MB", DEFAULT_MIN_FREE_MB, 0, 1_048_576), arrivalCutoffMinutes, arrivalEndMs: arrivalEndMs({ minutes, faultSpacingMinutes, arrivalCutoffMinutes }, options), + retryDuringWave: values["mix-retry-during-wave"] === true, + settleMinutes: whole(values["mix-settle-minutes"], "mix-settle-minutes", 15, 0, 120), ...(values["mix-tasks"] !== undefined ? { tasks: whole(values["mix-tasks"], "mix-tasks", 1, 1, 2_000) } : {}), }; if (mix.tasks === undefined && mix.arrivalEndMs < (mix.runs + mix.queue) * BURST_GAP_MS) { @@ -269,7 +281,8 @@ export function planStagingMixTasks(options: WaveOptions, rng = createRng((optio holdMs: 0, followUpAfterMs: null, nonce: rng.hex(8), - stagingMix: { turnMinutes, rounds, subtasks, followUpAfterMs, arriveAtMs, holdMs: mix.holdMinutes * 60_000 }, + stagingMix: { turnMinutes, rounds, subtasks, followUpAfterMs, arriveAtMs, holdMs: mix.holdMinutes * 60_000, + ...(mix.retryDuringWave ? { retryDuringWave: true } : {}) }, }; }); } @@ -354,6 +367,10 @@ export function stagingMixDescription(nonce: string, task: StagingMixTask, agent export const followUpRequestId = (issueId: string, nonce: string) => stableRequestId(`followup-${issueId}-${nonce}`); +export function isDoneComment(body: string, nonce: string): boolean { + return body.trim().replace(/\.$/, "").replace(/^`+|`+$/g, "").trim() === doneMarker(nonce); +} + export function followUpComment(nonce: string): string { return `Additional requirement: include the line \`${followUpMarker(nonce)}\` on its own line in this task's \`ledger\` document. Keep going with your rounds; do not restart any.`; } @@ -387,7 +404,9 @@ export function verifyLedger(nonce: string, rounds: number, text: string | null export function gradeStagingMix(nonce: string, task: StagingMixTask, artifacts: OracleArtifacts): OracleResult { const ledger = artifacts.documents.ledger ?? null; const own = verifyLedger(nonce, task.rounds, ledger); - const doneComments = artifacts.agentComments.filter(body => body.includes(doneMarker(nonce))).length; + const comments = artifacts.stagingMix?.comments ?? artifacts.agentComments.map(body => ({ body, runId: null })); + const doneComments = comments.filter(comment => isDoneComment(comment.body, nonce)).length; + const completionRuns = [...new Set(comments.filter(comment => comment.body.includes(doneMarker(nonce)) && comment.runId).map(comment => comment.runId!))]; const followUp = artifacts.stagingMix?.followUp ?? null; const children = (artifacts.children ?? []).filter(child => (child.title ?? "").includes(nonce)); const checks: OracleResult["checks"] = [ @@ -395,6 +414,8 @@ export function gradeStagingMix(nonce: string, task: StagingMixTask, artifacts: { name: "ledger_rounds_verified", ok: own.verified, detail: own.detail }, { name: "rounds_once", ok: own.duplicates === 0, detail: `duplicates=${own.duplicates}` }, { name: "done_comment_once", ok: doneComments === 1, detail: `comments=${doneComments}` }, + { name: "completion_once", ok: completionRuns.length <= 1, + detail: completionRuns.length > 1 ? `duplicate completion: runs ${completionRuns.map(id => id.slice(0, 8)).join(", ")} each reported completion` : `runs=${completionRuns.length}` }, ]; if (task.followUpAfterMs !== null) { const errors = artifacts.stagingMix?.followUpErrors ?? []; @@ -502,17 +523,46 @@ export async function stagingMixStep(input: StagingMixStepInput): Promise input.note("staging_mix_follow_up", { ...followUp }); } } + noteRetryBanner(state, retryShown(sample), now, latestFailureCode(sample)); if (!retryOffered(sample) || !retryDue(record)) { if (!retryOffered(sample)) state.retrySeenAt = null; return; } state.retrySeenAt ??= now; - if (!overlap) return; + if (!overlap || !task.retryDuringWave) return; (state.retries ??= []).push({ at: now, inOverlap: true }); input.note("staging_mix_retry_in_overlap", { waitedMs: now - state.retrySeenAt }); await input.retry(sample, now); } -export function stagingMixArtifacts(record: TaskRecord): StagingMixArtifacts { - return { followUp: record.stagingMix?.followUp ?? null, followUpErrors: record.stagingMix?.followUpErrors ?? [], retries: record.stagingMix?.retries ?? [] }; +export function retryShown(sample: Pick): boolean { + return sample.reachable && sample.issueStatus !== "done" && Boolean(sample.interruption?.canRetry); +} + +export function latestFailureCode(sample: Pick): string | null { + const failed = (sample.runs ?? []).filter(run => run.status === "failed" || run.status === "timed_out" || run.errorCode) + .sort((a, b) => Date.parse(b.createdAt ?? "") - Date.parse(a.createdAt ?? "")); + return failed[0]?.errorCode ?? failed[0]?.failureCause ?? null; +} + +export function noteRetryBanner(state: StagingMixRecord, shown: boolean, now: number, errorCode: string | null): void { + const banner = state.retryBanner; + if (!shown) { + if (banner?.visibleSince != null) { banner.shownMs += now - banner.visibleSince; banner.visibleSince = null; } + return; + } + if (!banner) { state.retryBanner = { firstShownAt: now, lastShownAt: now, shownMs: 0, visibleSince: now, errorCode }; return; } + banner.lastShownAt = now; + banner.visibleSince ??= now; + banner.errorCode = errorCode ?? banner.errorCode; +} + +export function bannerShownMs(banner: RetryBanner, now: number): number { + return banner.shownMs + (banner.visibleSince === null ? 0 : now - banner.visibleSince); +} + +export async function collectStagingMixArtifacts(api: Pick, issueId: string, record: TaskRecord): Promise { + const rows = await api.get>(`/api/issues/${issueId}/comments`).catch(() => null); + return { followUp: record.stagingMix?.followUp ?? null, followUpErrors: record.stagingMix?.followUpErrors ?? [], retries: record.stagingMix?.retries ?? [], + ...(rows ? { comments: rows.filter(row => row.authorAgentId).map(row => ({ body: String(row.body ?? ""), runId: row.createdByRunId ?? null })) } : {}) }; }