diff --git a/doc/plans/2026-11-11-staging-failures-vs-chaos-coverage.md b/doc/plans/2026-11-11-staging-failures-vs-chaos-coverage.md index 7c0bb57e730..25714e623fb 100644 --- a/doc/plans/2026-11-11-staging-failures-vs-chaos-coverage.md +++ b/doc/plans/2026-11-11-staging-failures-vs-chaos-coverage.md @@ -190,6 +190,11 @@ working hours is about 40 minutes. Every native run longer than about 40 minutes and OpenCode) passed 2 of 2 on master. The save-back did not block the task. The harness keeps no evidence for passing tasks, so it is not verified that the link existed at save-back time. - Proposed (not built): `DF1`, which fills the sandbox disk to ENOSPC before result copy-back. +- Built (2026-11-11): `DF1` (`--faults DF1`, `--env daytona`). A test-only server hook makes the + task's final saves report a full sandbox disk until the harness clears it, instead of filling a + real disk. See "Sandbox disk full (DF1)" in `tests/restart-chaos/README.md`. First wave + `w-20261111T131941-s9412` (Daytona, R1, Codex): the task rested with `sandbox_disk_full` and the + API offered Retry, but the board showed "Recover saved files" and no Retry button. ### E: Daytona capacity outage (18 runs, mostly self-healing) @@ -274,7 +279,7 @@ working hours is about 40 minutes. Every native run longer than about 40 minutes | K15 redeploy into a new process space and deployment identity | R, P, L (and A with `--upgrade-from`) | `--faults K15` | Implemented, unit-tested. Five Daytona waves are recorded below. | | EP1 environment-provider 5xx window | E | — | Proposed | | F1 absolute symlink kept in the repository workspace | F (unsafe-symlink save-back) | `--workloads F1` | Implemented, unit-tested. Daytona wave: 2 of 2 pass on master. | -| DF1 sandbox disk full | F (ENOSPC save-back) | — | Proposed | +| DF1 sandbox disk full | F (ENOSPC save-back) | — | Built (test-only save hook) | ### Waves on master 8f6ca25976 diff --git a/server/src/services/remote-workspaces.ts b/server/src/services/remote-workspaces.ts index 2920a4f6447..5e2ebec34a7 100644 --- a/server/src/services/remote-workspaces.ts +++ b/server/src/services/remote-workspaces.ts @@ -9,6 +9,7 @@ import os from "node:os"; import path from "node:path"; import { and, eq, inArray, isNull, or, sql } from "drizzle-orm"; import { crashPoint } from "@paperclipai/shared/crash-points"; +import { sandboxDiskFullFaultTarget } from "./sandbox-disk-full-fault.js"; import { companies, environmentLeases, executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issues, workspaceOperations, type Db, } from "@paperclipai/db"; @@ -912,7 +913,9 @@ export async function saveRemoteWorkspace(input: { const existingUpload = snapshot.status === "uploaded" ? await getWorkspaceSnapshot(input.db, { companyId: input.workspace.companyId, workspaceId: input.workspace.id, snapshotId: snapshot.id }) : null; if (existingUpload) timing.reusedUpload = true; - captured = existingUpload ? null : await captureRemoteWorkspaceArchive({ target, remoteCwd: expected.location.remoteCwd, + captured = existingUpload ? null : await captureRemoteWorkspaceArchive({ remoteCwd: expected.location.remoteCwd, + // Test-only DF1 hook (sandbox-disk-full-fault.ts): the target itself unless a harness armed this save. + target: sandboxDiskFullFaultTarget(target, { boundary, issueId: operation.issueId, runId: input.runId }), operationId: operation.id, timing }); await input.assertOwned(); const uploadStarted = performance.now(); diff --git a/server/src/services/sandbox-disk-full-fault.test.ts b/server/src/services/sandbox-disk-full-fault.test.ts new file mode 100644 index 00000000000..af2e2ae3a3c --- /dev/null +++ b/server/src/services/sandbox-disk-full-fault.test.ts @@ -0,0 +1,78 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { REMOTE_WORKSPACE_SANDBOX_DISK_FULL, captureRemoteWorkspaceArchive, type RemoteWorkspaceTarget } from "./remote-workspace-transport.js"; +import { + DISK_FULL_FAULT_CONTROL_ENV, DISK_FULL_FAULT_RECEIPT_ENV, DISK_FULL_FAULT_SYNC_OUT_MESSAGE, + readDiskFullFaultGate, sandboxDiskFullFaultTarget, setDiskFullFaultGateForTest, +} from "./sandbox-disk-full-fault.js"; + +const ISSUE = "11111111-1111-4111-8111-111111111111"; +const RUN = "22222222-2222-4222-8222-222222222222"; +const OPERATION = "33333333-3333-4333-8333-333333333333"; + +function sandboxTarget(calls: string[]): RemoteWorkspaceTarget { + return { + kind: "remote", transport: "sandbox", remoteCwd: "/workspace", providerKey: "test", + runner: { + execute: async () => { calls.push("execute"); return { exitCode: 0, signal: null, timedOut: false, stdout: "{}", stderr: "", pid: 1, startedAt: null }; }, + syncIn: async () => { calls.push("syncIn"); return {} as never; }, + syncOut: async () => { calls.push("syncOut"); return {} as never; }, + }, + } as unknown as RemoteWorkspaceTarget; +} + +describe("sandbox disk-full fault gate", () => { + let restore: (() => void) | null = null; + const dirs: string[] = []; + afterEach(() => { + restore?.(); restore = null; + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); + }); + const scratch = () => { const dir = mkdtempSync(path.join(os.tmpdir(), "disk-full-fault-")); dirs.push(dir); return dir; }; + + it("is off unless the harness switch is set, and never on in production", () => { + expect(readDiskFullFaultGate({})).toBeNull(); + expect(readDiskFullFaultGate({ [DISK_FULL_FAULT_CONTROL_ENV]: " " })).toBeNull(); + expect(readDiskFullFaultGate({ NODE_ENV: "production", [DISK_FULL_FAULT_CONTROL_ENV]: "/tmp/control" })).toBeNull(); + expect(readDiskFullFaultGate({ NODE_ENV: "test", [DISK_FULL_FAULT_CONTROL_ENV]: "/tmp/control", [DISK_FULL_FAULT_RECEIPT_ENV]: "/tmp/receipt" })) + .toEqual({ controlPath: "/tmp/control", receiptPath: "/tmp/receipt" }); + }); + + it("returns the save's own target when the process was started without the switch", () => { + const target = sandboxTarget([]); + expect(sandboxDiskFullFaultTarget(target, { boundary: "final", issueId: ISSUE, runId: RUN })).toBe(target); + }); + + it("faults only a final save whose issue or run is listed, and stops when cleared", async () => { + const dir = scratch(); + const control = path.join(dir, "control"), receipt = path.join(dir, "receipts.jsonl"); + restore = setDiskFullFaultGateForTest({ controlPath: control, receiptPath: receipt }); + const calls: string[] = []; + const target = sandboxTarget(calls); + // No control file yet: disarmed. + expect(sandboxDiskFullFaultTarget(target, { boundary: "final", issueId: ISSUE, runId: RUN })).toBe(target); + writeFileSync(control, `other-issue\n${ISSUE}\n`); + expect(sandboxDiskFullFaultTarget(target, { boundary: "initial", issueId: ISSUE, runId: RUN })).toBe(target); + expect(sandboxDiskFullFaultTarget(target, { boundary: "final", issueId: "other", runId: "other-run" })).toBe(target); + + const faulted = sandboxDiskFullFaultTarget(target, { boundary: "final", issueId: ISSUE, runId: RUN }); + expect(faulted).not.toBe(target); + expect(await faulted.runner!.execute({ command: "node" })).toMatchObject({ exitCode: 1, stderr: REMOTE_WORKSPACE_SANDBOX_DISK_FULL }); + await expect(faulted.runner!.syncOut!([])).rejects.toThrow(DISK_FULL_FAULT_SYNC_OUT_MESSAGE); + await faulted.runner!.syncIn!([]); + expect(calls).toEqual(["syncIn"]); + expect(JSON.parse(readFileSync(receipt, "utf8").trim())).toMatchObject({ issueId: ISSUE, runId: RUN, boundary: "final" }); + + // The save's own capture path turns the sandbox's answer into the named disk-full failure. + await expect(captureRemoteWorkspaceArchive({ target: faulted, remoteCwd: "/workspace", operationId: OPERATION })) + .rejects.toThrow(REMOTE_WORKSPACE_SANDBOX_DISK_FULL); + + // A run id arms it too; clearing the file disarms the next save. + writeFileSync(control, `${RUN}\n`); + expect(sandboxDiskFullFaultTarget(target, { boundary: "final", issueId: null, runId: RUN })).not.toBe(target); + writeFileSync(control, ""); + expect(sandboxDiskFullFaultTarget(target, { boundary: "final", issueId: ISSUE, runId: RUN })).toBe(target); + }); +}); diff --git a/server/src/services/sandbox-disk-full-fault.ts b/server/src/services/sandbox-disk-full-fault.ts new file mode 100644 index 00000000000..0e57e22b9df --- /dev/null +++ b/server/src/services/sandbox-disk-full-fault.ts @@ -0,0 +1,82 @@ +/** + * Test-only sandbox disk-full fault (restart-chaos DF1, tests/restart-chaos/disk-full-faults.ts). + * + * A real full sandbox disk cannot be made on demand, locally or on Daytona. This hook makes a + * final workspace save see what a full sandbox returns, and leaves every decision after that to + * the product: the save's own failure handling, finalization, saved-work recovery and Retry. + * + * Arming (test mode only), in the same style as `@paperclipai/shared/crash-points`: + * + * PAPERCLIP_DISK_FULL_FAULT_CONTROL= each final save reads this file. A line that equals + * the save's issue id or run id arms that save. No file, + * or no matching line, means disarmed. + * PAPERCLIP_DISK_FULL_FAULT_RECEIPT= optional; one JSON line is appended per faulted save. + * + * While a save is armed, every sandbox command its capture runs exits 1 with the exact message the + * in-sandbox capture script writes on ENOSPC, and the provider's file download fails with tar's + * "No space left on device". Restore and other operations are untouched. + * + * The variable is read once, when this module is first imported, and is ignored when + * `NODE_ENV=production`. Disarmed, the hook is one comparison against a module constant and + * returns the save's own target. Only the chaos harness sets the variable, on the server it starts. + * + * This is test instrumentation, not Telemetry or Observability: nothing is recorded or sent + * anywhere unless a test arms a save. + */ +import { appendFileSync, readFileSync } from "node:fs"; +import { REMOTE_WORKSPACE_SANDBOX_DISK_FULL, type RemoteWorkspaceTarget } from "./remote-workspace-transport.js"; + +export const DISK_FULL_FAULT_CONTROL_ENV = "PAPERCLIP_DISK_FULL_FAULT_CONTROL"; +export const DISK_FULL_FAULT_RECEIPT_ENV = "PAPERCLIP_DISK_FULL_FAULT_RECEIPT"; +/** The provider download's failure on a full disk: tar cannot write its archive. */ +export const DISK_FULL_FAULT_SYNC_OUT_MESSAGE = "tar: workspace archive: Cannot write: No space left on device"; + +export type DiskFullFaultReceipt = { issueId: string | null; runId: string; boundary: string; pid: number; at: string }; + +type Gate = { controlPath: string; receiptPath: string | null }; + +export function readDiskFullFaultGate(env: NodeJS.ProcessEnv | undefined): Gate | null { + if (!env || env.NODE_ENV === "production") return null; + const controlPath = env[DISK_FULL_FAULT_CONTROL_ENV]?.trim(); + if (!controlPath) return null; + return { controlPath, receiptPath: env[DISK_FULL_FAULT_RECEIPT_ENV]?.trim() || null }; +} + +let gate: Gate | null = readDiskFullFaultGate(typeof process === "undefined" ? undefined : process.env); +if (gate) process.stderr.write(`[disk-full-fault] armed from ${gate.controlPath} pid=${process.pid}\n`); + +function armed(current: Gate, ids: readonly string[]): boolean { + let text: string; + try { text = readFileSync(current.controlPath, "utf8"); } catch { return false; } + const lines = new Set(text.split("\n").map(line => line.trim()).filter(Boolean)); + return ids.some(id => lines.has(id)); +} + +/** + * The target a workspace save captures through. Disarmed, or for any boundary other than the + * final copy-back, it is `target` itself. + */ +export function sandboxDiskFullFaultTarget(target: T, + save: { boundary: string; issueId: string | null; runId: string }): T { + if (gate === null) return target; + if (save.boundary !== "final" || !armed(gate, [save.runId, ...(save.issueId ? [save.issueId] : [])])) return target; + const receipt: DiskFullFaultReceipt = { issueId: save.issueId, runId: save.runId, boundary: save.boundary, + pid: process.pid, at: new Date().toISOString() }; + if (gate.receiptPath) { + try { appendFileSync(gate.receiptPath, `${JSON.stringify(receipt)}\n`); } catch { /* the log line still records it */ } + } + process.stderr.write(`[disk-full-fault] save ${JSON.stringify(receipt)}\n`); + const runner = target.runner!; + const full = Object.create(runner) as typeof runner; + full.execute = async () => ({ exitCode: 1, signal: null, timedOut: false, stdout: "", + stderr: REMOTE_WORKSPACE_SANDBOX_DISK_FULL, pid: null, startedAt: null }); + if (runner.syncOut) full.syncOut = async () => { throw new Error(DISK_FULL_FAULT_SYNC_OUT_MESSAGE); }; + return { ...target, runner: full }; +} + +/** Replaces the gate inside a test process; returns a function that restores the previous one. */ +export function setDiskFullFaultGateForTest(next: Gate | null): () => void { + const previous = gate; + gate = next; + return () => { gate = previous; }; +} diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 2de445fb04e..4539c747378 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -414,6 +414,53 @@ How the product saves the link itself (skip it, keep it, or record it) is not graded. F1 is meaningful with a remote workspace (`--env daytona`, with `--storage s3` as on staging). Locally it is a plain repository task. +## Sandbox disk full (DF1) + +```sh +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona --daytona-image $IMG --daytona-cpu 3 \ + --adapters codex --workloads R1 --workspace-repo "$PWD" --faults DF1 --storage s3 --concurrency 2 --tasks 2 --seed 9412 +``` + +On staging, saving a run's result back from its sandbox failed because the sandbox disk was full +(census class F, `doc/plans/2026-11-11-staging-failures-vs-chaos-coverage.md`). The product then +ends saved-work recovery at once with cause `sandbox_disk_full` and offers a Retry (#20758). + +A full disk cannot be made on demand, so DF1 uses a test-only hook in the server +(`server/src/services/sandbox-disk-full-fault.ts`). The harness starts its server with +`PAPERCLIP_DISK_FULL_FAULT_CONTROL=` and `PAPERCLIP_DISK_FULL_FAULT_RECEIPT=/disk-full-faults.jsonl`. +The hook reads the variable once at startup and ignores it when `NODE_ENV=production`. Without it the +hook is one comparison and the save uses its own target. + +- **Armed**: when a DF1 task launches, the harness writes its issue id to the control file. Each + final workspace save of that task (finalization's copy-back and recovery's re-save) then gets what + a full sandbox returns: every capture command exits 1 with the product's disk-full message, and + the provider download fails with `No space left on device`. The product's own code does the rest. + The server appends one receipt per faulted save. +- **Cleared**: when the task rests with the cause `sandbox_disk_full` (or is `done`), the harness + records what the board shows, removes the id, and then the observer clicks Retry, as a person who + freed space would. A board-owned recovery can look like a rest while it still checks the saved + source; the harness keeps waiting there for up to 10 minutes (`DISK_FULL_REST_SETTLE_MS`) before it + clears the fault anyway, so that recovery's own save cannot hide the disk-full rest. + +The task passes when: + +- the fault reached at least one save (a receipt for the task); +- it rested with recovery cause `sandbox_disk_full` and a Retry offered (T check `disk_full_fault`); +- exactly one Retry, after the fault cleared, finished the task (R3). A needed Retry is expected + here, so R3 passes on it. Whether that Retry starts a new run is not graded. + +The ledger line records `diskFull: { faultedSaves, restedDiskFull, retryOffered, retryWorked }`. + +DF1 restarts nothing and needs `--env daytona --storage s3`: only a remote workspace has a sandbox +save, so the harness refuses other setups. Use R1 with `--workspace-repo`. With the chaoscli fixture +(B1) on Daytona, the workspace is a local worktree synced into the sandbox, so no remote save runs +and the task fails as `not_applied` (wave `w-20261111T131314-s9411`). An unchanged workspace also +skips the capture. + +The observer clicks Retry in the board UI, as for every other cell. In waves `w-20261111T131941-s9412` +and `w-20261111T134316-s9413` the disk-full rest showed "Recover saved files", not a Retry button, while the API reported +`canRetry: true`; R3 records this as `ui_retry_button_missing`. + ## Config faults (CF1–CF3, `config-wave.ts`) ```sh @@ -1138,6 +1185,7 @@ re-arms the task's own triggered faults instead of using their offsets. | `journeys.ts`, `journey-driver.ts`, `journey-oauth-app.ts`, `journey-matrix.ts` | Q2 and Q3 definitions and oracles, the board's side of each, Q3's loopback OAuth/MCP app, and the `--matrix` planner. | | `evidence.ts`, `ledger.ts` | Evidence bundle for each failure and the JSONL ledger plus `SUMMARY.md`. | | `daytona-image.ts` | Docker-free extraction of the image's runnerd and provider pack, cached per digest, and the controller/image content-ID check. | +| `disk-full-faults.ts` | DF1: the control file for the server's test-only disk-full hook, the rest evidence, and the `disk_full_fault` check and R3 rule. | | `provider-faults.ts`, `provider-shim/` | PV1–PV3: the resolve hook, the Codex stdio shim, per-task rules and the `provider_fault_applied` check. | | `acpx-provider-faults.ts`, `provider-shim/acpx-*` | PV1–PV3 for Claude (ACPX): the stand-in runner package, the server hook, the sidecar/provider tap, and `--provider-fault-restart`. | | `question-handoff.ts` | Q1 saved question: the board's answer step and its oracle. | diff --git a/tests/restart-chaos/config.ts b/tests/restart-chaos/config.ts index 6e9ab880600..eae8f75317a 100644 --- a/tests/restart-chaos/config.ts +++ b/tests/restart-chaos/config.ts @@ -13,6 +13,7 @@ import { parseRepeatedRestartOptions } from "./repeated-restarts.js"; import { daytonaRetainsSandboxes } from "./ttft-baseline-stats.js"; import { PROVIDER_FAULT_KINDS, PROVIDER_FAULT_WORKLOADS, isProviderFault, parseProviderDelays, parseProviderMessages, type ProviderFaultKind } from "./provider-faults.js"; import { PROVIDER_FAULT_ADAPTERS, parseProviderFaultRestart } from "./acpx-provider-faults.js"; +import { DISK_FULL_FAULT_KINDS, isDiskFullFault, type DiskFullFaultKind } from "./disk-full-faults.js"; import { ADAPTER_IDS, ENVIRONMENT_KINDS, FAULT_KINDS, TRIGGER_IDS, WORKLOAD_IDS, type AdapterId, type EnvironmentKind, type FaultKind, type TriggerId, type WaveOptions, type WorkloadId, @@ -174,10 +175,12 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC } if (lifecycle !== undefined && lifecycle !== "per_turn" && lifecycle !== "warm" && lifecycle !== "default") throw new Error("--lifecycle must be per_turn, warm or default"); const adapters = list(values.adapters, ADAPTER_IDS, "adapters", ["codex", "claude"]); - // `--faults` takes restart faults (K*) and provider message faults (PV*, provider-faults.ts). - const allFaults = list(values.faults, [...FAULT_KINDS, ...PROVIDER_FAULT_KINDS], "faults", + // `--faults` takes restart faults (K*), provider message faults (PV*, provider-faults.ts) and the + // sandbox disk-full fault (DF1, disk-full-faults.ts). + const allFaults = list(values.faults, [...FAULT_KINDS, ...PROVIDER_FAULT_KINDS, ...DISK_FULL_FAULT_KINDS], "faults", stormMinutes ? STORM_DEFAULT_FAULTS : crashPoint ? ["K1"] : ["K1", "K2", "K4"]); const providerFaults = allFaults.filter(isProviderFault); + const diskFullFaults = allFaults.filter(isDiskFullFault); const options: WaveOptions = { env, adapters, @@ -185,8 +188,9 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC workloads: list(values.workloads, workloadChoices, "workloads", workloadChoices.filter(id => id !== "G1" && id !== "H1" && id !== "R1" && id !== "S1" && id !== "L1" && id !== "F1" && id !== "T1" && !takesJourneyTriggers(id) && (id !== "M1" || adapters.length >= 2))), // A crash-point wave schedules no fault; K1 is the restart its self-kill gets. - faults: allFaults.filter((fault): fault is FaultKind => !isProviderFault(fault)), + faults: allFaults.filter((fault): fault is FaultKind => !isProviderFault(fault) && !isDiskFullFault(fault)), ...(providerFaults.length ? { providerFaults } : {}), + ...(diskFullFaults.length ? { diskFullFaults } : {}), triggers: list(values.triggers, TRIGGER_IDS, "triggers", ["time"]), concurrency: integer(values.concurrency, "concurrency", 4, 1, 20), tasks: integer(values.tasks, "tasks", stormMinutes ? 200 : 8, 1, 200), @@ -215,6 +219,14 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC throw new Error("--fault-after-seconds fires faults by time; use it with --triggers time"); } if (!crashPoint && values["crash-point-hold-ms"] !== undefined) throw new Error("--crash-point-hold-ms applies only to --crash-point"); + if (diskFullFaults.length) { + if (stormMinutes) throw new Error("The disk-full fault (DF1) is planned per task; do not combine it with --storm-minutes"); + // Only a remote sandbox workspace has a save the sandbox disk can refuse; a local run has none. + if (env !== "daytona" || values.storage !== "s3") throw new Error("The disk-full fault (DF1) needs a remote sandbox workspace save; run it with --env daytona --storage s3"); + if (providerFaults.length) throw new Error("The disk-full fault (DF1) does not share a wave with provider message faults (PV*)"); + // The matrix planner builds its cells from restart and provider faults only; DF1 would silently drop out. + if (values.matrix) throw new Error("The disk-full fault (DF1) is not a --matrix cell; run it as a planned wave"); + } if (providerFaults.length) { if (stormMinutes) throw new Error("Provider message faults (PV*) are planned per task; do not combine them with --storm-minutes"); if (env !== "local") throw new Error("Provider message faults (PV*) shim the local provider; run them with --env local"); diff --git a/tests/restart-chaos/disk-full-faults.test.ts b/tests/restart-chaos/disk-full-faults.test.ts new file mode 100644 index 00000000000..f6bc0273840 --- /dev/null +++ b/tests/restart-chaos/disk-full-faults.test.ts @@ -0,0 +1,113 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { parseCommand } from "./config.js"; +import { DISK_FULL_REST_SETTLE_MS, DiskFullFaultSeam, diskFullLedger, diskFullRest, diskFullRestReady, receiptsForIssue, withDiskFullCheck } from "./disk-full-faults.js"; +import { gradeR3 } from "./grader.js"; +import { planTasks, taskFaultSchedule } from "./scheduler.js"; +import type { TaskRecord } from "./types.js"; + +const IMAGE = `ghcr.io/paperclipai/paperclip-daytona-runner@sha256:${"a".repeat(64)}`; +const diskFullIssue = (label = "sandbox_disk_full", canRetry = true) => ({ + status: "blocked", interruption: { canRetry }, + activeRecoveryAction: { ownerType: "board", evidence: { systemRecovery: { failure: { + code: "export_interrupted", retryClass: "user_action_required", cause: { label, code: "ENOSPC", status: null } } } } }, +}); + +function record(extra: Partial = {}): TaskRecord { + return { + plan: { taskId: "t01", index: 0, workload: "B1", adapter: "codex", fault: "DF1", fireAfterMs: 0, holdMs: 0, followUpAfterMs: null, nonce: "abcd1234" }, + issueId: "issue-1", identifier: "CHA-1", createdAt: 0, launchError: null, samples: [], retry: null, restAt: 1, + finalStatus: "done", deadlineAt: 1, timedOut: false, followUp: null, faultIds: [], sweep: null, + preFaultFiles: null, lostFiles: null, + diskFull: { kind: "DF1", armedAt: 1, clearedAt: 2, rest: diskFullRest(diskFullIssue(), 2), appliedSaves: 2 }, + ...extra, + }; +} +const accepted = { at: 3, observedRunId: "run-1", httpStatus: 200, accepted: true, error: null }; +const passing = { verdict: "pass" as const, checks: [] }; + +describe("DF1 planning", () => { + it("is a per-task fault on Daytona that schedules no restart", () => { + const command = parseCommand(["--env", "daytona", "--daytona-image", IMAGE, "--adapters", "codex", "--workloads", "B1", + "--faults", "DF1", "--storage", "s3", "--tasks", "2", "--seed", "7"]); + if (command.mode !== "wave") throw new Error("expected a wave"); + expect(command.options).toMatchObject({ faults: [], diskFullFaults: ["DF1"] }); + const tasks = planTasks(command.options); + expect(tasks.map(task => task.fault)).toEqual(["DF1", "DF1"]); + expect(tasks.map(taskFaultSchedule)).toEqual([[], []]); + }); + + it("refuses waves where no sandbox save exists or the plan cannot carry it", () => { + expect(() => parseCommand(["--adapters", "codex", "--workloads", "B1", "--faults", "DF1"])).toThrow(/--env daytona/); + expect(() => parseCommand(["--env", "daytona", "--daytona-image", IMAGE, "--adapters", "codex", "--workloads", "B1", "--faults", "DF1"])).toThrow(/--storage s3/); + expect(() => parseCommand(["--env", "daytona", "--daytona-image", IMAGE, "--adapters", "codex", "--faults", "DF1", "--storage", "s3", "--storm-minutes", "5"])) + .toThrow(/storm/); + expect(() => parseCommand(["--env", "daytona", "--daytona-image", IMAGE, "--adapters", "codex", "--workloads", "B1", "--faults", "DF1", "--storage", "s3", "--matrix"])) + .toThrow(/--matrix/); + }); +}); + +describe("DF1 control file", () => { + let root: string | null = null; + afterEach(async () => { if (root) await rm(root, { recursive: true, force: true }); root = null; }); + + it("arms and clears issue ids atomically and reads the server's receipts", async () => { + root = await mkdtemp(path.join(os.tmpdir(), "df1-seam-")); + const receiptPath = path.join(root, "receipts.jsonl"); + const seam = await DiskFullFaultSeam.install({ root, receiptPath }); + expect(seam.serverEnvironment).toEqual({ PAPERCLIP_DISK_FULL_FAULT_CONTROL: seam.controlPath, PAPERCLIP_DISK_FULL_FAULT_RECEIPT: receiptPath }); + expect(await readFile(seam.controlPath, "utf8")).toBe(""); + await Promise.all([seam.arm("issue-1"), seam.arm("issue-2")]); + expect(await readFile(seam.controlPath, "utf8")).toBe("issue-1\nissue-2\n"); + await seam.clear("issue-1"); + expect(await readFile(seam.controlPath, "utf8")).toBe("issue-2\n"); + await writeFile(receiptPath, `${JSON.stringify({ issueId: "issue-2", runId: "r", boundary: "final", pid: 1, at: "t" })}\nnot json\n`); + expect(receiptsForIssue(await seam.receipts(), "issue-2")).toHaveLength(1); + expect(receiptsForIssue(await seam.receipts(), "issue-1")).toHaveLength(0); + }); +}); + +describe("DF1 clear timing", () => { + it("waits at an unfinished rest until the board names the full disk, then up to the settle bound", () => { + const disk = { kind: "DF1" as const, armedAt: 1, clearedAt: null, rest: null }; + const checking = diskFullRest({ status: "blocked", interruption: { canRetry: true }, activeRecoveryAction: { ownerType: "board", evidence: {} } }, 0); + expect(diskFullRestReady(disk, checking, 1_000)).toBe(false); + expect(diskFullRestReady(disk, checking, 1_000 + DISK_FULL_REST_SETTLE_MS - 1)).toBe(false); + expect(diskFullRestReady(disk, diskFullRest(diskFullIssue(), 0), 2_000)).toBe(true); + expect(diskFullRestReady(disk, { ...checking, status: "done" }, 2_000)).toBe(true); + expect(diskFullRestReady(disk, checking, 1_000 + DISK_FULL_REST_SETTLE_MS)).toBe(true); + }); +}); + +describe("DF1 grading", () => { + it("passes when the task rested on a full disk with a Retry and one Retry finished it", () => { + const graded = record({ retry: accepted }); + expect(withDiskFullCheck(passing, graded.plan, graded).verdict).toBe("pass"); + expect(gradeR3(graded)).toMatchObject({ verdict: "pass" }); + expect(diskFullLedger(graded, "done")).toEqual({ faultedSaves: 2, restedDiskFull: true, retryOffered: true, retryWorked: true }); + }); + + it("fails a fault that never reached a save, a different rest cause, or a missing Retry", () => { + const notApplied = record({ diskFull: { kind: "DF1", armedAt: 1, clearedAt: null, rest: null, appliedSaves: 0 } }); + expect(withDiskFullCheck(passing, notApplied.plan, notApplied).checks[0]).toMatchObject({ ok: false, detail: expect.stringContaining("not_applied") }); + const otherCause = record({ retry: accepted, diskFull: { kind: "DF1", armedAt: 1, clearedAt: 2, rest: diskFullRest(diskFullIssue("provider_error"), 2), appliedSaves: 1 } }); + expect(withDiskFullCheck(passing, otherCause.plan, otherCause).checks[0]?.detail).toContain("rest_cause=provider_error"); + const noRetryOffered = record({ diskFull: { kind: "DF1", armedAt: 1, clearedAt: 2, rest: diskFullRest(diskFullIssue("sandbox_disk_full", false), 2), appliedSaves: 1 } }); + expect(withDiskFullCheck(passing, noRetryOffered.plan, noRetryOffered).checks[0]?.detail).toContain("retry_not_offered"); + expect(gradeR3(record({ finalStatus: "blocked" }))).toMatchObject({ verdict: "fail", detail: expect.stringContaining("disk_full_retry_missing") }); + }); + + it("fails a Retry that did not finish the task or needed a second click", () => { + expect(gradeR3(record({ retry: accepted, finalStatus: "blocked" }))).toMatchObject({ verdict: "fail", detail: expect.stringContaining("did_not_complete") }); + expect(gradeR3(record({ retry: accepted, priorRetries: [accepted] }))).toMatchObject({ verdict: "fail", detail: expect.stringContaining("2_clicks") }); + expect(gradeR3(record({ retry: { ...accepted, accepted: false } }))).toMatchObject({ verdict: "fail", detail: expect.stringContaining("retry_rejected") }); + }); + + it("leaves other cells to the ordinary Retry rule", () => { + const plain = record({ plan: { ...record().plan, fault: "K1" }, retry: accepted, diskFull: undefined }); + expect(withDiskFullCheck(passing, plain.plan, plain)).toBe(passing); + expect(gradeR3(plain)).toMatchObject({ verdict: "fail", detail: expect.stringContaining("retry_no_new_run") }); + }); +}); diff --git a/tests/restart-chaos/disk-full-faults.ts b/tests/restart-chaos/disk-full-faults.ts new file mode 100644 index 00000000000..10d7cc5f8ad --- /dev/null +++ b/tests/restart-chaos/disk-full-faults.ts @@ -0,0 +1,183 @@ +/** + * DF1: the sandbox disk is full when the run's result is saved (staging census class F, + * doc/plans/2026-11-11-staging-failures-vs-chaos-coverage.md). + * + * A genuinely full disk cannot be made on demand, so the server carries a test-only hook + * (server/src/services/sandbox-disk-full-fault.ts). The harness starts its server with + * PAPERCLIP_DISK_FULL_FAULT_CONTROL pointing at a file it owns. While a task's issue id is in + * that file, each final workspace save of that task gets the answer a full sandbox gives: the + * capture command exits 1 with the product's disk-full message, and the provider download fails + * with "No space left on device". Everything after that is the product's own handling. + * + * The harness arms the task when it launches and clears it when the task first rests offering a + * Retry, just before the observer clicks Retry, as a person who freed space would. Grading: + * - the fault applied (the server wrote a receipt for the task); + * - the task rested with recovery cause `sandbox_disk_full` and a Retry offered; + * - one Retry, after the fault cleared, finished the task (R3). + * + * DF1 restarts nothing. It needs a remote sandbox workspace, so it runs with `--env daytona`. + */ +import { randomUUID } from "node:crypto"; +import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { OracleResult, RuleResult, TaskPlan, TaskRecord } from "./types.js"; + +type Json = Record; + +export const DISK_FULL_FAULT_KINDS = ["DF1"] as const; +export type DiskFullFaultKind = (typeof DISK_FULL_FAULT_KINDS)[number]; +export function isDiskFullFault(value: string): value is DiskFullFaultKind { + return (DISK_FULL_FAULT_KINDS as readonly string[]).includes(value); +} + +/** The recovery cause the product names for a save that found the sandbox disk full. */ +export const SANDBOX_DISK_FULL_CAUSE = "sandbox_disk_full"; + +/** Server-side receipt, one per faulted save (sandbox-disk-full-fault.ts DiskFullFaultReceipt). */ +export interface DiskFullReceipt { issueId: string | null; runId: string; boundary: string; pid: number; at: string } + +/** What the board showed when a DF1 task first rested. */ +export interface DiskFullRest { + at: number; + status: string | null; + canRetry: boolean; + /** `activeRecoveryAction.evidence.systemRecovery.failure.cause.label`. */ + causeLabel: string | null; + retryClass: string | null; + ownerType: string | null; +} + +export interface DiskFullRecord { + kind: DiskFullFaultKind; + armedAt: number | null; + clearedAt: number | null; + rest: DiskFullRest | null; + /** First rest seen while armed; the harness waits for the disk-full verdict from here (DISK_FULL_REST_SETTLE_MS). */ + firstRestAt?: number; + /** Faulted saves the server recorded for the task; filled in at grading. */ + appliedSaves?: number; + armError?: string; +} + +export function diskFullRest(issue: Json | null, at: number): DiskFullRest { + const action = (issue?.activeRecoveryAction ?? null) as Json | null; + const failure = (action?.evidence?.systemRecovery?.failure ?? null) as Json | null; + return { + at, status: typeof issue?.status === "string" ? issue.status : null, + canRetry: issue?.interruption?.canRetry === true, + causeLabel: typeof failure?.cause?.label === "string" ? failure.cause.label : null, + retryClass: typeof failure?.retryClass === "string" ? failure.retryClass : null, + ownerType: typeof action?.ownerType === "string" ? action.ownerType : null, + }; +} + +/** + * A board-owned recovery can look like a rest while it is still checking the saved source + * ("Recovering saved files…", wave w-20261111T131941-s9412/t02). Clearing the fault there would + * let that recovery's own save succeed and hide the disk-full rest. While armed, the harness + * keeps waiting at an unfinished rest until the board names `sandbox_disk_full`, up to this long. + */ +export const DISK_FULL_REST_SETTLE_MS = 10 * 60_000; + +/** True when the harness may record this rest and clear the fault; false keeps watching. */ +export function diskFullRestReady(record: DiskFullRecord, rest: DiskFullRest, now: number): boolean { + if (rest.status === "done" || rest.causeLabel === SANDBOX_DISK_FULL_CAUSE) return true; + record.firstRestAt ??= now; + return now - record.firstRestAt >= DISK_FULL_REST_SETTLE_MS; +} + +/** The control file the server reads and the receipts it writes. */ +export class DiskFullFaultSeam { + private readonly armed = new Set(); + private writes: Promise = Promise.resolve(); + + private constructor(readonly controlPath: string, readonly receiptPath: string) {} + + static async install(input: { root: string; receiptPath: string }): Promise { + const directory = path.join(input.root, "disk-full-fault"); + await mkdir(directory, { recursive: true, mode: 0o700 }); + const seam = new DiskFullFaultSeam(path.join(directory, "control"), input.receiptPath); + await seam.write(); + return seam; + } + + get serverEnvironment(): Record { + return { PAPERCLIP_DISK_FULL_FAULT_CONTROL: this.controlPath, PAPERCLIP_DISK_FULL_FAULT_RECEIPT: this.receiptPath }; + } + + arm(issueId: string): Promise { this.armed.add(issueId); return this.write(); } + clear(issueId: string): Promise { this.armed.delete(issueId); return this.write(); } + + /** Atomic, so a save never reads half a file; serialized, so the last change wins. */ + private write(): Promise { + const body = [...this.armed].map(id => `${id}\n`).join(""); + this.writes = this.writes.catch(() => undefined).then(async () => { + const temporary = `${this.controlPath}.${randomUUID()}.tmp`; + await writeFile(temporary, body, { mode: 0o600 }); + await rename(temporary, this.controlPath); + }); + return this.writes; + } + + async receipts(): Promise { + const text = await readFile(this.receiptPath, "utf8").catch(() => ""); + return text.split("\n").filter(Boolean).flatMap(line => { try { return [JSON.parse(line) as DiskFullReceipt]; } catch { return []; } }); + } +} + +export function receiptsForIssue(receipts: readonly DiskFullReceipt[], issueId: string | null): DiskFullReceipt[] { + return issueId ? receipts.filter(receipt => receipt.issueId === issueId) : []; +} + +/** Adds the `disk_full_fault` check: the fault applied and the task rested on it with a Retry. */ +export function withDiskFullCheck(oracle: OracleResult, plan: TaskPlan, record: TaskRecord): OracleResult { + if (!isDiskFullFault(plan.fault)) return oracle; + const disk = record.diskFull; + const applied = disk?.appliedSaves ?? 0; + const rest = disk?.rest ?? null; + const problems = [ + ...(disk?.armError ? [`arm_failed=${disk.armError}`] : []), + ...(applied === 0 ? ["not_applied (no save reached the hook: an unchanged workspace, a local run or a server without the hook)"] : []), + ...(applied > 0 && !rest ? ["no_rest_while_armed"] : []), + ...(rest && rest.causeLabel !== SANDBOX_DISK_FULL_CAUSE ? [`rest_cause=${rest.causeLabel ?? "none"}`] : []), + ...(rest && !rest.canRetry ? ["retry_not_offered"] : []), + ]; + const detail = `${plan.fault}: faultedSaves=${applied} rest=${rest ? `${rest.status}/${rest.causeLabel ?? "none"}/${rest.retryClass ?? "none"}/canRetry=${rest.canRetry}` : "none"}` + + (problems.length ? `; ${problems.join("; ")}` : ""); + const checks = [...oracle.checks, { name: "disk_full_fault", ok: problems.length === 0, detail }]; + return { verdict: checks.every(check => check.ok) ? "pass" : "fail", checks }; +} + +/** + * R3 for DF1. The product cannot free a sandbox disk, so one Retry is the expected outcome, not a + * failure: R3 passes when exactly one accepted Retry, after the fault cleared, finished the task. + * Whether the Retry started a new run is not graded: applying the already-saved result without a + * new turn is a correct Retry too. + */ +export function gradeDiskFullRetry(record: TaskRecord, expectedStatus: string): RuleResult { + const retry = record.retry; + if (!retry) { + return { verdict: "fail", detail: `disk_full_retry_missing: rested ${record.finalStatus ?? "unknown"}${record.timedOut ? " (timed out)" : ""} without a Retry` }; + } + if (retry.error || retry.accepted !== true) { + return { verdict: "fail", detail: `retry_rejected: http=${retry.httpStatus ?? "none"} accepted=${String(retry.accepted)} ${retry.error ?? ""}`.trim() }; + } + const clicks = 1 + (record.priorRetries?.length ?? 0); + if (clicks > 1) return { verdict: "fail", detail: `retry_needed_${clicks}_clicks (expected one Retry after the disk-full fault cleared)` }; + if (record.finalStatus !== expectedStatus) { + return { verdict: "fail", detail: `disk_full_retry_did_not_complete (status ${record.finalStatus ?? "unknown"}${record.timedOut ? ", timed out" : ""})` }; + } + return { verdict: "pass", detail: `expected Retry after ${SANDBOX_DISK_FULL_CAUSE}; Retry worked (${expectedStatus})` }; +} + +/** Ledger fields: needed-Retry and Retry-worked for the cell. */ +export function diskFullLedger(record: TaskRecord, expectedStatus: string) { + const disk = record.diskFull; + if (!disk) return undefined; + return { + faultedSaves: disk.appliedSaves ?? 0, + restedDiskFull: disk.rest?.causeLabel === SANDBOX_DISK_FULL_CAUSE, + retryOffered: disk.rest?.canRetry === true, + retryWorked: record.retry?.accepted === true && record.finalStatus === expectedStatus, + }; +} diff --git a/tests/restart-chaos/grader.ts b/tests/restart-chaos/grader.ts index 1a3730f456e..fd8e0f93893 100644 --- a/tests/restart-chaos/grader.ts +++ b/tests/restart-chaos/grader.ts @@ -1,5 +1,6 @@ import { UI_RETRY_BLOCKED_BY_MODAL, UI_RETRY_BUTTON_MISSING, UI_RETRY_NO_REQUEST, type FaultReceipt, type OracleResult, type RuleId, type RuleResult, type TaskGrade, type TaskRecord, type TaskSample } from "./types.js"; import { g1Decision } from "./workloads.js"; +import { gradeDiskFullRetry, isDiskFullFault } from "./disk-full-faults.js"; import { enforcedUniversalFailures, gradeUniversal, type UniversalDbEvidence } from "./universal.js"; /** Pure R1–R6 grading over observer samples (plan §2). No I/O. */ @@ -98,7 +99,8 @@ export function gradeR2(record: TaskRecord, oracle: OracleResult): RuleResult { } export function gradeR3(record: TaskRecord): RuleResult { - const own = gradeOwnRetry(record); + // DF1: a full sandbox disk needs a person, so one Retry is expected (disk-full-faults.ts). + const own = isDiskFullFault(record.plan.fault) ? gradeDiskFullRetry(record, expectedFinalStatus(record)) : gradeOwnRetry(record); const childClicks = record.childRetries?.length ?? 0; if (!childClicks || own.verdict === "fail") return own; return { verdict: "fail", detail: `retry_needed_on_subtask_${childClicks} (${record.childRetries!.map(attempt => attempt.childIssueId).join(",")})` }; diff --git a/tests/restart-chaos/ledger.ts b/tests/restart-chaos/ledger.ts index d527fdc558b..894a23f6f46 100644 --- a/tests/restart-chaos/ledger.ts +++ b/tests/restart-chaos/ledger.ts @@ -2,6 +2,8 @@ import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises"; import path from "node:path"; import { replayCommand } from "./config.js"; import type { RuleId, TaskGrade, TaskRecord, WavePlan } from "./types.js"; +import { diskFullLedger } from "./disk-full-faults.js"; +import { expectedFinalStatus } from "./grader.js"; export interface LedgerLine { schema: "paperclip.restart-chaos.ledger/v1"; @@ -37,6 +39,8 @@ export interface LedgerLine { /** Universal checks U1–U4 (universal.ts): verdict per rule, and the details of warnings that did not fail the task. */ universal?: Record; universalWarnings?: string[]; + /** DF1: faulted saves, the disk-full rest with a Retry offered, and whether that Retry worked. */ + diskFull?: { faultedSaves: number; restedDiskFull: boolean; retryOffered: boolean; retryWorked: boolean }; } export function ledgerLine(plan: WavePlan, record: TaskRecord, grade: TaskGrade, gitSha: string, evidence: string | null, @@ -55,6 +59,7 @@ export function ledgerLine(plan: WavePlan, record: TaskRecord, grade: TaskGrade, recordedAt: new Date().toISOString(), ...(plan.options.masks?.length ? { masks: [...plan.options.masks] } : {}), ...(plan.options.crashPoint ? { crashPoint: plan.options.crashPoint } : {}), + ...(record.diskFull ? { diskFull: diskFullLedger(record, expectedFinalStatus(record)) } : {}), ...(grade.universal ? { universal: Object.fromEntries(Object.entries(grade.universal).map(([id, rule]) => [id, rule.verdict])), universalWarnings: Object.entries(grade.universal).filter(([, rule]) => !rule.enforced && rule.verdict === "fail") diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index 86dcb0f05ec..131eafc10dd 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -23,6 +23,7 @@ import { NativeTrigger, nativeTriggerMarkdown, nativeTriggerShortfall, triggered import { needsCredentialRotation, startCredentialRotation, type CredentialRotation } from "./storage-credentials.js"; import { describeProviderFault, installProviderFaultSeam, resolveRealCodex, withProviderFaultCheck, type ProviderFaultSeam } from "./provider-faults.js"; import { installAcpxSidecarSeam, withAcpxSidecarHook, type AcpxSidecarSeam } from "./acpx-provider-faults.js"; +import { DiskFullFaultSeam, isDiskFullFault, receiptsForIssue, withDiskFullCheck } from "./disk-full-faults.js"; import { CREDENTIAL_FOR_ADAPTER, parseCommand, replayCommand, waveIdFor } from "./config.js"; import { assertControllerMatchesImage, defaultDaytonaImageCacheRoot, prepareDaytonaImageArtifacts } from "./daytona-image.js"; import { PLUGIN_BUILD_STAMP, pluginBuildState } from "./plugin-build.js"; @@ -274,6 +275,7 @@ async function main(): Promise { let storageAuditor: StorageAuditor | null = null; let providerFaults: ProviderFaultSeam | null = null; let acpxSidecarSeam: AcpxSidecarSeam | null = null; + let diskFullSeam: DiskFullFaultSeam | null = null; let speedSection: string | null = null; let storageSection: string | null = null; let harnessError: unknown; @@ -336,6 +338,12 @@ async function main(): Promise { timeline.add("provider_faults_installed", { realCodex: realCodex.command, codexVersion: realCodex.version, rules: plan.tasks.flatMap(task => task.providerFault ? [`${task.taskId}:${describeProviderFault(task.providerFault)}`] : []) }); } + if (plan.tasks.some(task => isDiskFullFault(task.fault))) { + // DF1: the server's test-only hook reads this control file at each final workspace save. + diskFullSeam = await DiskFullFaultSeam.install({ root: supervisor.paths.temporaryRoot, receiptPath: path.join(waveDir, "disk-full-faults.jsonl") }); + Object.assign(serverEnvironment, diskFullSeam.serverEnvironment); + timeline.add("disk_full_faults_installed", { tasks: plan.tasks.filter(task => isDiskFullFault(task.fault)).map(task => task.taskId) }); + } if (plan.options.timeAgeMs) supervisor.timeAgeMs = plan.options.timeAgeMs; if (plan.options.rrStopOn) supervisor.rrStopOn = plan.options.rrStopOn; await supervisor.start(serverEnvironment); @@ -369,6 +377,7 @@ async function main(): Promise { board = serverEnvironment.SERVE_UI === "false" ? null : new BoardUi(supervisor.baseUrl, undefined, path.join(waveDir, "ui-retry-diagnostics")); const observers = plan.tasks.map(task => new TaskObserver(task, api, lab, timeline, supervisor, protectedRoots, board)); + for (const observer of observers) if (isDiskFullFault(observer.plan.fault)) observer.diskFullSeam = diskFullSeam; const pending = [...observers]; const launched: TaskObserver[] = []; const active: TaskObserver[] = []; @@ -678,8 +687,13 @@ async function main(): Promise { const restart = faults.find(receipt => receipt.requestedBy === observer.plan.taskId && receipt.trigger === "provider"); const oracle = withProviderFaultCheck(graded, observer.plan, artifacts?.providerFaultEvents ?? [], restart?.startedAt ?? null, restart?.stoppedAt ?? restart?.startedAt ?? null); + if (observer.record.diskFull) { + observer.record.diskFull.appliedSaves = receiptsForIssue(await diskFullSeam?.receipts() ?? [], observer.record.issueId).length; + } // T1: a mid-tool fault only counts when one of the task's faults started while its loop ran (workloads.ts). - const toolOracle = withT1CrossingCheck(oracle, observer.plan, artifacts?.documents.brief, faults); + // DF1 restarts nothing, so T1's mid-tool restart check does not apply; the disk-full check replaces it. + const toolOracle = isDiskFullFault(observer.plan.fault) ? withDiskFullCheck(oracle, observer.plan, observer.record) + : withT1CrossingCheck(oracle, observer.plan, artifacts?.documents.brief, faults); const grade = gradeTask(observer.record, faults, observer.definition.budgetMs, toolOracle, universalByIssue ? { runMessages: allRunMessages, recoveryActions: universalByIssue.get(observer.record.issueId ?? "")?.recoveryActions ?? [] } : undefined); let evidence: string | null = null; diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index 8618de508c1..2a74d8bcc4c 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -21,6 +21,7 @@ import { taskFaultSchedule } from "./scheduler.js"; import { g1ChildTitle, g1Decision, workloadDefinitions, type WorkloadAgents } from "./workloads.js"; import { databaseUrlFromConfig, journeyDriverFor, type JourneyDriver } from "./journey-driver.js"; import { withLiveStatusEvidence } from "./live-status-evidence.js"; +import { diskFullRest, diskFullRestReady, isDiskFullFault, type DiskFullFaultSeam } from "./disk-full-faults.js"; const IGNORED_DIRECTORIES = new Set([".git", ".bench", "__pycache__", "node_modules"]); const MAX_SNAPSHOT_FILES = 2_000; @@ -207,6 +208,8 @@ export class TaskObserver { private readonly faultSchedule: ScheduledFault[]; private readonly pendingFaults: ScheduledFault[]; done = false; + /** DF1 (disk-full-faults.ts): arms this task's saves at launch and clears them before its Retry. */ + diskFullSeam: Pick | null = null; /** Replaceable in tests. A Daytona run's process is remote, so only local runs are checked. */ processAlive = (pid: number, startedAt: string) => this.lab.environmentKind === "local" && localProcessAlive(pid, startedAt); @@ -288,6 +291,7 @@ export class TaskObserver { this.record.deadlineAt = this.record.createdAt + definition.budgetMs; this.timeline.add("task_launched", { taskId: this.plan.taskId, issueId: issue.id, identifier: issue.identifier, workload: this.plan.workload, adapter: this.plan.adapter, faults: this.faultSchedule }); + await this.armDiskFull(); } catch (error) { this.record.launchError = error instanceof Error ? error.message : String(error); this.done = true; @@ -605,6 +609,37 @@ export class TaskObserver { return true; } + /** DF1: the task's final saves find the sandbox disk full until the fault is cleared. */ + private async armDiskFull(): Promise { + if (!isDiskFullFault(this.plan.fault) || !this.record.issueId) return; + const record = this.record.diskFull = { kind: this.plan.fault, armedAt: null as number | null, clearedAt: null as number | null, rest: null }; + try { + if (!this.diskFullSeam) throw new Error("disk-full seam not installed"); + await this.diskFullSeam.arm(this.record.issueId); + record.armedAt = Date.now(); + this.timeline.add("disk_full_fault_armed", { taskId: this.plan.taskId, issueId: this.record.issueId }); + } catch (error) { + (record as NonNullable).armError = error instanceof Error ? error.message : String(error); + this.timeline.add("disk_full_fault_arm_failed", { taskId: this.plan.taskId, error: (record as NonNullable).armError }); + } + } + + /** + * DF1: once the rest names the full disk (or the task is done, or the settle bound passed), record + * what the board shows, then clear the fault, as a person who freed space. Returns false to keep watching. + */ + private async clearDiskFullAtRest(now: number): Promise { + const record = this.record.diskFull; + if (!record || record.armedAt === null || record.clearedAt !== null || !this.record.issueId) return true; + const rest = diskFullRest(this.issue, now); + if (!diskFullRestReady(record, rest, now)) return false; + record.rest = rest; + await this.diskFullSeam!.clear(this.record.issueId); + record.clearedAt = Date.now(); + this.timeline.add("disk_full_fault_cleared", { taskId: this.plan.taskId, rest: record.rest }); + return true; + } + private async onRest(sample: TaskSample, now: number) { const status = sample.issueStatus ?? null; // The board has not acted yet: the task waits on it, so it has not rested (and gets no Retry). @@ -629,6 +664,7 @@ export class TaskObserver { return; } } + if (!await this.clearDiskFullAtRest(now)) { this.restCandidateSince = null; return; } if (status !== "done" && (!this.record.retry || faultSinceRetry) && sample.interruption?.canRetry) { if (this.record.retry) (this.record.priorRetries ??= []).push(this.record.retry); this.record.retry = await this.retry(sample, now); diff --git a/tests/restart-chaos/scheduler.ts b/tests/restart-chaos/scheduler.ts index 3ba1b1f4796..fdde7f8865a 100644 --- a/tests/restart-chaos/scheduler.ts +++ b/tests/restart-chaos/scheduler.ts @@ -1,4 +1,5 @@ import { isProviderFault, withProviderFaultRules } from "./provider-faults.js"; +import { isDiskFullFault } from "./disk-full-faults.js"; import { takesJourneyTriggers } from "./journeys.js"; import { workloadDefinitions } from "./workloads.js"; import { isTimeFault, timeFaultHoldMs } from "./time-faults-config.js"; @@ -79,8 +80,8 @@ const STORM_RESULT_FAULTS: readonly FaultKind[] = ["K1", "K2"]; * A `result` task in a storm also fires its own K1 or K2 just after its run * accepts a result, the window of RC-30, RC-33 and RC-41. */ -function stormCellFaults(options: Pick, trigger: TriggerId): readonly TaskFaultKind[] { - if (!options.stormMinutes) return [...options.faults, ...(options.providerFaults ?? [])]; +function stormCellFaults(options: Pick, trigger: TriggerId): readonly TaskFaultKind[] { + if (!options.stormMinutes) return [...options.faults, ...(options.providerFaults ?? []), ...(options.diskFullFaults ?? [])]; if (trigger === "time") return ["K13"]; if (trigger !== "result") throw new Error(`A K13 storm fires on its own clock; --triggers ${trigger} does not combine with it (use time or result)`); const faults = options.faults.filter(fault => STORM_RESULT_FAULTS.includes(fault)); @@ -96,7 +97,7 @@ function pinnedFireAfterMs(faultAfterMs: number | undefined, drawn: number): num return faultAfterMs ?? drawn; } -export function planTasks(options: Pick): TaskPlan[] { +export function planTasks(options: Pick): TaskPlan[] { const rng = createRng(options.seed); const triggers = options.triggers?.length ? options.triggers : ["time" as const]; for (const trigger of triggers) { @@ -293,11 +294,11 @@ export function observedFaultSchedule(createdAt: number | null, faultIds: readon /** The faults a task schedules: its replayed sequence, or its one planned fault. */ export function taskFaultSchedule(task: TaskPlan): ScheduledFault[] { if (task.replayFaults?.length) return task.replayFaults; - // A storm crosses K13 tasks; a provider fault (PV*) and a matrix baseline (K0) restart nothing, unless + // A storm crosses K13 tasks; a provider fault (PV*), a disk-full fault (DF1) and a matrix baseline (K0) restart nothing, unless // `--provider-fault-restart` asked for a K1 once the provider fault applied (the `provider` trigger). const restart = task.providerFault?.restart; if (restart) return [{ kind: restart.kind, holdMs: restart.holdMs ?? 0, atMs: 0, trigger: "provider", delayMs: restart.afterAppliedMs }]; - if (task.fault === "K13" || task.fault === "K0" || isProviderFault(task.fault)) return []; + if (task.fault === "K13" || task.fault === "K0" || isProviderFault(task.fault) || isDiskFullFault(task.fault)) return []; const planned: ScheduledFault = { kind: task.fault, holdMs: task.holdMs, atMs: task.fireAfterMs, ...(task.restartSteps ? { restartSteps: task.restartSteps } : {}) }; const trigger = task.trigger ?? "time"; diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index 3622401ecf3..9e3cf8a48be 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -1,4 +1,5 @@ /** Shared types for the control-plane restart chaos harness (plan §5). */ +import type { DiskFullFaultKind, DiskFullRecord } from "./disk-full-faults.js"; import type { ProviderFaultEvent, ProviderFaultKind, ProviderFaultRule, ProviderMessage, ProviderRestartKind } from "./provider-faults.js"; import type { QuestionArtifacts, QuestionRecord } from "./question-handoff.js"; import type { RepeatedRestartRecord, RestartStep } from "./repeated-restarts.js"; @@ -20,8 +21,9 @@ export const K4_MIN_HOLD_MS = 120_000; * A task's planned fault. "K13" (restart storm) tasks have none of their own; the wave's storm crosses them. * PV1–PV3 (provider message faults, provider-faults.ts) restart nothing: the task's rule is armed in the Codex shim. * "K0" is a `--matrix` baseline cell: the same journey with no fault. + * DF1 (disk-full-faults.ts) restarts nothing either: the task's final workspace saves find the sandbox disk full. */ -export type TaskFaultKind = FaultKind | "K13" | "K0" | ProviderFaultKind; +export type TaskFaultKind = FaultKind | "K13" | "K0" | ProviderFaultKind | DiskFullFaultKind; /** * G1: an approval-gated action across a restart. H1: a budget hard stop crossed mid-run with a restart (its own wave). @@ -117,6 +119,8 @@ export interface WaveOptions { workspaceGitUrl?: string; /** Provider message faults from `--faults` (PV1 delay, PV2 drop, PV3 duplicate); see provider-faults.ts. */ providerFaults?: ProviderFaultKind[]; + /** Sandbox disk-full faults from `--faults` (DF1); see disk-full-faults.ts. */ + diskFullFaults?: DiskFullFaultKind[]; providerFaultMessage?: ProviderMessage; /** `--provider-fault-message S1=interrupt_ack,...`: a workload's own message, over `providerFaultMessage`. */ providerFaultMessageByWorkload?: Partial>; @@ -419,6 +423,8 @@ export interface TaskRecord { journey?: import("./journeys.js").JourneyRecord; /** PV with `--provider-fault-restart`: when the shim logged the task's fault (arms the `provider` trigger). */ providerFaultAppliedAt?: number; + /** DF1: when the fault was armed and cleared, and what the board showed at the disk-full rest. */ + diskFull?: DiskFullRecord; } export interface TaskGrade {