From 626f552e88ea3af2d3a229457dcf076ce938f201 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 11 Nov 2026 07:42:55 -0600 Subject: [PATCH] test(chaos): wait for the disk-full verdict before clearing DF1; require a remote workspace A board-owned recovery that is still checking its saved source looked like a rest, so DF1 cleared too early and that recovery's save succeeded. The harness now waits at an unfinished rest for the sandbox_disk_full cause, up to 10 minutes. DF1 also needs --storage s3, and the docs point at R1, since the chaoscli fixture syncs a local worktree and runs no remote save. Co-Authored-By: Claude --- ...1-11-staging-failures-vs-chaos-coverage.md | 4 +++- tests/restart-chaos/README.md | 21 +++++++++++++------ tests/restart-chaos/config.ts | 2 +- tests/restart-chaos/disk-full-faults.test.ts | 19 ++++++++++++++--- tests/restart-chaos/disk-full-faults.ts | 17 +++++++++++++++ tests/restart-chaos/observer.ts | 18 ++++++++++------ 6 files changed, 64 insertions(+), 17 deletions(-) diff --git a/doc/plans/2026-11-11-staging-failures-vs-chaos-coverage.md b/doc/plans/2026-11-11-staging-failures-vs-chaos-coverage.md index fce07ac9437..25714e623fb 100644 --- a/doc/plans/2026-11-11-staging-failures-vs-chaos-coverage.md +++ b/doc/plans/2026-11-11-staging-failures-vs-chaos-coverage.md @@ -192,7 +192,9 @@ working hours is about 40 minutes. Every native run longer than about 40 minutes - Proposed (not built): `DF1`, which fills the sandbox disk to ENOSPC before result copy-back. - Built (2026-11-11): `DF1` (`--faults DF1`, `--env daytona`). A test-only server hook makes the task's final saves report a full sandbox disk until the harness clears it, instead of filling a - real disk. See "Sandbox disk full (DF1)" in `tests/restart-chaos/README.md`. + real disk. See "Sandbox disk full (DF1)" in `tests/restart-chaos/README.md`. First wave + `w-20261111T131941-s9412` (Daytona, R1, Codex): the task rested with `sandbox_disk_full` and the + API offered Retry, but the board showed "Recover saved files" and no Retry button. ### E: Daytona capacity outage (18 runs, mostly self-healing) diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 64bf10fb0e1..ebd2f1eb416 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -418,7 +418,7 @@ graded. F1 is meaningful with a remote workspace (`--env daytona`, with ```sh node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona --daytona-image $IMG --daytona-cpu 3 \ - --adapters codex --workloads B1 --faults DF1 --storage s3 --concurrency 2 --tasks 2 --seed 9401 + --adapters codex --workloads R1 --workspace-repo "$PWD" --faults DF1 --storage s3 --concurrency 2 --tasks 2 --seed 9412 ``` On staging, saving a run's result back from its sandbox failed because the sandbox disk was full @@ -436,8 +436,11 @@ hook is one comparison and the save uses its own target. a full sandbox returns: every capture command exits 1 with the product's disk-full message, and the provider download fails with `No space left on device`. The product's own code does the rest. The server appends one receipt per faulted save. -- **Cleared**: when the task first rests, the harness records what the board shows, removes the id, - and then the observer clicks Retry, as a person who freed space would. +- **Cleared**: when the task rests with the cause `sandbox_disk_full` (or is `done`), the harness + records what the board shows, removes the id, and then the observer clicks Retry, as a person who + freed space would. A board-owned recovery can look like a rest while it still checks the saved + source; the harness keeps waiting there for up to 10 minutes (`DISK_FULL_REST_SETTLE_MS`) before it + clears the fault anyway, so that recovery's own save cannot hide the disk-full rest. The task passes when: @@ -448,9 +451,15 @@ The task passes when: The ledger line records `diskFull: { faultedSaves, restedDiskFull, retryOffered, retryWorked }`. -DF1 restarts nothing and needs `--env daytona`: a local run has no sandbox workspace save, so the -harness refuses `--env local`. Use workloads that change workspace files (B1, F1, R1, T1). An -unchanged workspace skips the capture and fails as `not_applied`. +DF1 restarts nothing and needs `--env daytona --storage s3`: only a remote workspace has a sandbox +save, so the harness refuses other setups. Use R1 with `--workspace-repo`. With the chaoscli fixture +(B1) on Daytona, the workspace is a local worktree synced into the sandbox, so no remote save runs +and the task fails as `not_applied` (wave `w-20261111T131314-s9411`). An unchanged workspace also +skips the capture. + +The observer clicks Retry in the board UI, as for every other cell. In wave `w-20261111T131941-s9412` +the disk-full rest showed "Recover saved files", not a Retry button, while the API reported +`canRetry: true`; R3 records this as `ui_retry_button_missing`. ## Config faults (CF1–CF3, `config-wave.ts`) diff --git a/tests/restart-chaos/config.ts b/tests/restart-chaos/config.ts index fc0926c3314..aa4e0bfe329 100644 --- a/tests/restart-chaos/config.ts +++ b/tests/restart-chaos/config.ts @@ -222,7 +222,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC if (diskFullFaults.length) { if (stormMinutes) throw new Error("The disk-full fault (DF1) is planned per task; do not combine it with --storm-minutes"); // Only a remote sandbox workspace has a save the sandbox disk can refuse; a local run has none. - if (env !== "daytona") throw new Error("The disk-full fault (DF1) needs a remote sandbox workspace save; run it with --env daytona"); + if (env !== "daytona" || values.storage !== "s3") throw new Error("The disk-full fault (DF1) needs a remote sandbox workspace save; run it with --env daytona --storage s3"); if (providerFaults.length) throw new Error("The disk-full fault (DF1) does not share a wave with provider message faults (PV*)"); } if (providerFaults.length) { diff --git a/tests/restart-chaos/disk-full-faults.test.ts b/tests/restart-chaos/disk-full-faults.test.ts index e3527e52bd5..b1808b7d45f 100644 --- a/tests/restart-chaos/disk-full-faults.test.ts +++ b/tests/restart-chaos/disk-full-faults.test.ts @@ -3,7 +3,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { parseCommand } from "./config.js"; -import { DiskFullFaultSeam, diskFullLedger, diskFullRest, receiptsForIssue, withDiskFullCheck } from "./disk-full-faults.js"; +import { DISK_FULL_REST_SETTLE_MS, DiskFullFaultSeam, diskFullLedger, diskFullRest, diskFullRestReady, receiptsForIssue, withDiskFullCheck } from "./disk-full-faults.js"; import { gradeR3 } from "./grader.js"; import { planTasks, taskFaultSchedule } from "./scheduler.js"; import type { TaskRecord } from "./types.js"; @@ -31,7 +31,7 @@ const passing = { verdict: "pass" as const, checks: [] }; describe("DF1 planning", () => { it("is a per-task fault on Daytona that schedules no restart", () => { const command = parseCommand(["--env", "daytona", "--daytona-image", IMAGE, "--adapters", "codex", "--workloads", "B1", - "--faults", "DF1", "--tasks", "2", "--seed", "7"]); + "--faults", "DF1", "--storage", "s3", "--tasks", "2", "--seed", "7"]); if (command.mode !== "wave") throw new Error("expected a wave"); expect(command.options).toMatchObject({ faults: [], diskFullFaults: ["DF1"] }); const tasks = planTasks(command.options); @@ -41,7 +41,8 @@ describe("DF1 planning", () => { it("refuses waves where no sandbox save exists or the plan cannot carry it", () => { expect(() => parseCommand(["--adapters", "codex", "--workloads", "B1", "--faults", "DF1"])).toThrow(/--env daytona/); - expect(() => parseCommand(["--env", "daytona", "--daytona-image", IMAGE, "--adapters", "codex", "--faults", "DF1", "--storm-minutes", "5"])) + expect(() => parseCommand(["--env", "daytona", "--daytona-image", IMAGE, "--adapters", "codex", "--workloads", "B1", "--faults", "DF1"])).toThrow(/--storage s3/); + expect(() => parseCommand(["--env", "daytona", "--daytona-image", IMAGE, "--adapters", "codex", "--faults", "DF1", "--storage", "s3", "--storm-minutes", "5"])) .toThrow(/storm/); }); }); @@ -66,6 +67,18 @@ describe("DF1 control file", () => { }); }); +describe("DF1 clear timing", () => { + it("waits at an unfinished rest until the board names the full disk, then up to the settle bound", () => { + const disk = { kind: "DF1" as const, armedAt: 1, clearedAt: null, rest: null }; + const checking = diskFullRest({ status: "blocked", interruption: { canRetry: true }, activeRecoveryAction: { ownerType: "board", evidence: {} } }, 0); + expect(diskFullRestReady(disk, checking, 1_000)).toBe(false); + expect(diskFullRestReady(disk, checking, 1_000 + DISK_FULL_REST_SETTLE_MS - 1)).toBe(false); + expect(diskFullRestReady(disk, diskFullRest(diskFullIssue(), 0), 2_000)).toBe(true); + expect(diskFullRestReady(disk, { ...checking, status: "done" }, 2_000)).toBe(true); + expect(diskFullRestReady(disk, checking, 1_000 + DISK_FULL_REST_SETTLE_MS)).toBe(true); + }); +}); + describe("DF1 grading", () => { it("passes when the task rested on a full disk with a Retry and one Retry finished it", () => { const graded = record({ retry: accepted }); diff --git a/tests/restart-chaos/disk-full-faults.ts b/tests/restart-chaos/disk-full-faults.ts index 3105a957bd8..10d7cc5f8ad 100644 --- a/tests/restart-chaos/disk-full-faults.ts +++ b/tests/restart-chaos/disk-full-faults.ts @@ -52,6 +52,8 @@ export interface DiskFullRecord { armedAt: number | null; clearedAt: number | null; rest: DiskFullRest | null; + /** First rest seen while armed; the harness waits for the disk-full verdict from here (DISK_FULL_REST_SETTLE_MS). */ + firstRestAt?: number; /** Faulted saves the server recorded for the task; filled in at grading. */ appliedSaves?: number; armError?: string; @@ -69,6 +71,21 @@ export function diskFullRest(issue: Json | null, at: number): DiskFullRest { }; } +/** + * A board-owned recovery can look like a rest while it is still checking the saved source + * ("Recovering saved files…", wave w-20261111T131941-s9412/t02). Clearing the fault there would + * let that recovery's own save succeed and hide the disk-full rest. While armed, the harness + * keeps waiting at an unfinished rest until the board names `sandbox_disk_full`, up to this long. + */ +export const DISK_FULL_REST_SETTLE_MS = 10 * 60_000; + +/** True when the harness may record this rest and clear the fault; false keeps watching. */ +export function diskFullRestReady(record: DiskFullRecord, rest: DiskFullRest, now: number): boolean { + if (rest.status === "done" || rest.causeLabel === SANDBOX_DISK_FULL_CAUSE) return true; + record.firstRestAt ??= now; + return now - record.firstRestAt >= DISK_FULL_REST_SETTLE_MS; +} + /** The control file the server reads and the receipts it writes. */ export class DiskFullFaultSeam { private readonly armed = new Set(); diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index 3195fe2bf6d..2a74d8bcc4c 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -21,7 +21,7 @@ import { taskFaultSchedule } from "./scheduler.js"; import { g1ChildTitle, g1Decision, workloadDefinitions, type WorkloadAgents } from "./workloads.js"; import { databaseUrlFromConfig, journeyDriverFor, type JourneyDriver } from "./journey-driver.js"; import { withLiveStatusEvidence } from "./live-status-evidence.js"; -import { diskFullRest, isDiskFullFault, type DiskFullFaultSeam } from "./disk-full-faults.js"; +import { diskFullRest, diskFullRestReady, isDiskFullFault, type DiskFullFaultSeam } from "./disk-full-faults.js"; const IGNORED_DIRECTORIES = new Set([".git", ".bench", "__pycache__", "node_modules"]); const MAX_SNAPSHOT_FILES = 2_000; @@ -624,14 +624,20 @@ export class TaskObserver { } } - /** DF1: at the first rest, record what the board shows, then clear the fault, as a person who freed space. */ - private async clearDiskFullAtRest(now: number): Promise { + /** + * DF1: once the rest names the full disk (or the task is done, or the settle bound passed), record + * what the board shows, then clear the fault, as a person who freed space. Returns false to keep watching. + */ + private async clearDiskFullAtRest(now: number): Promise { const record = this.record.diskFull; - if (!record || record.armedAt === null || record.clearedAt !== null || !this.record.issueId) return; - record.rest = diskFullRest(this.issue, now); + if (!record || record.armedAt === null || record.clearedAt !== null || !this.record.issueId) return true; + const rest = diskFullRest(this.issue, now); + if (!diskFullRestReady(record, rest, now)) return false; + record.rest = rest; await this.diskFullSeam!.clear(this.record.issueId); record.clearedAt = Date.now(); this.timeline.add("disk_full_fault_cleared", { taskId: this.plan.taskId, rest: record.rest }); + return true; } private async onRest(sample: TaskSample, now: number) { @@ -658,7 +664,7 @@ export class TaskObserver { return; } } - await this.clearDiskFullAtRest(now); + if (!await this.clearDiskFullAtRest(now)) { this.restCandidateSince = null; return; } if (status !== "done" && (!this.record.retry || faultSinceRetry) && sample.interruption?.canRetry) { if (this.record.retry) (this.record.priorRetries ??= []).push(this.record.retry); this.record.retry = await this.retry(sample, now);