diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index d1189d1aadf..7066ddf8a2e 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -25,6 +25,8 @@ Options: `--triggers time,comment,retry,queued,tool,result` (fault timing, below `--workloads A1,B1,B5,C1`, `--budget-cents` (company hard cap, default 3000), `--poll-ms` (2000–5000), `--keep` (keep the temporary instance), `--k4-hold-max-seconds` (raise the seeded K4 hold upper end from 300 s, up to the 600 s outage cap). +`--fault-after-seconds ` fires every planned fault `n` seconds after its task launches (5–3600; with `--triggers time`), keeping +the rest of the seeded plan, for example to restart a Daytona R1 task after its workspace preparation. Each wave's temporary instance (embedded Postgres, workspaces, often 5–13 GB) is deleted when the wave ends, pass or fail, including after an interrupt. Evidence diff --git a/tests/restart-chaos/config.ts b/tests/restart-chaos/config.ts index d0cf0e1e2c6..72db19379c6 100644 --- a/tests/restart-chaos/config.ts +++ b/tests/restart-chaos/config.ts @@ -95,6 +95,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC "storm-minutes": { type: "string" }, lifecycle: { type: "string" }, "k4-hold-max-seconds": { type: "string" }, + "fault-after-seconds": { type: "string" }, "crash-point": { type: "string" }, "crash-point-hold-ms": { type: "string" }, "upgrade-from": { type: "string" }, @@ -108,7 +109,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC const pollMs = integer(values["poll-ms"], "poll-ms", 3_000, 2_000, 5_000); const budgetCents = integer(values["budget-cents"], "budget-cents", 3_000, 1, 100_000); if (values.replay !== undefined) { - for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "daytona-reuse-lease", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point", "crash-point-hold-ms", "upgrade-from", "workspace-repo", "storage", "storage-faults", "storage-fault-seconds"] as const) { + for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "daytona-reuse-lease", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point", "crash-point-hold-ms", "upgrade-from", "workspace-repo", "storage", "storage-faults", "storage-fault-seconds", "fault-after-seconds"] as const) { if (values[key] !== undefined) throw new Error(`--replay reproduces the saved plan; do not combine it with --${key}`); } return { mode: "replay", ref: parseReplayRef(values.replay), keep, @@ -163,10 +164,18 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC // Seeded K4 hold upper end; the controller-outage cap is 600 s (#19797). ...(values["k4-hold-max-seconds"] !== undefined ? { k4HoldMaxMs: integer(values["k4-hold-max-seconds"], "k4-hold-max-seconds", 300, 120, 600) * 1000 } : {}), + ...(values["fault-after-seconds"] !== undefined + ? { faultAfterMs: integer(values["fault-after-seconds"], "fault-after-seconds", 60, 5, 3_600) * 1000 } : {}), ...(crashPoint ? { crashPoint } : {}), ...(crashPoint && values["crash-point-hold-ms"] !== undefined ? { crashPointHoldMs: integer(values["crash-point-hold-ms"], "crash-point-hold-ms", 0, 0, 600_000) } : {}), }; + if (values["fault-after-seconds"] !== undefined && (stormMinutes || crashPoint)) { + throw new Error("--fault-after-seconds sets planned fault times; a storm or crash-point wave has none"); + } + if (values["fault-after-seconds"] !== undefined && (options.triggers ?? []).some(trigger => trigger !== "time")) { + throw new Error("--fault-after-seconds fires faults by time; use it with --triggers time"); + } if (!crashPoint && values["crash-point-hold-ms"] !== undefined) throw new Error("--crash-point-hold-ms applies only to --crash-point"); const upgradeFrom = values["upgrade-from"]; if (options.faults.includes("K14") !== (upgradeFrom !== undefined)) { diff --git a/tests/restart-chaos/scheduler.test.ts b/tests/restart-chaos/scheduler.test.ts index 9d5128d3488..b19b04c99fa 100644 --- a/tests/restart-chaos/scheduler.test.ts +++ b/tests/restart-chaos/scheduler.test.ts @@ -36,6 +36,28 @@ describe("scheduler determinism", () => { } }); + it("fires every planned fault at --fault-after-seconds without changing the rest of the seeded plan", () => { + const base = parseCommand(["--faults", "K1,K2", "--tasks", "6", "--seed", "77"]); + const pinned = parseCommand(["--faults", "K1,K2", "--tasks", "6", "--seed", "77", "--fault-after-seconds", "150"]); + if (base.mode !== "wave" || pinned.mode !== "wave") throw new Error("expected a wave"); + expect(pinned.options.faultAfterMs).toBe(150_000); + const planned = planTasks(pinned.options); + expect(planned.map(task => task.fireAfterMs)).toEqual(Array(6).fill(150_000)); + expect(planned.map(({ fireAfterMs: _fire, ...rest }) => rest)).toEqual(planTasks(base.options).map(({ fireAfterMs: _fire, ...rest }) => rest)); + expect(() => parseCommand(["--fault-after-seconds", "2"])).toThrow(/fault-after-seconds/); + expect(() => parseCommand(["--storm-minutes", "10", "--fault-after-seconds", "60"])).toThrow(/storm/); + expect(() => parseCommand(["--triggers", "time,comment", "--fault-after-seconds", "60"])).toThrow(/--triggers time/); + }); + + it("keeps the plans existing seeds produce for triggered and storm waves", () => { + const triggered = planTasks({ seed: 4242, tasks: 4, workloads: ["A1", "C1"], adapters: ["codex", "claude"], faults: ["K1", "K2"], + triggers: ["time", "comment", "result"] } as never); + expect(triggered.map(task => [task.trigger ?? "time", task.triggerDelayMs ?? null, task.fireAfterMs, task.nonce])).toEqual([ + ["time", null, 36292, "d3a12313"], ["time", null, 14380, "37697e51"], ["comment", 4272, 23204, "a5d0f179"], ["time", null, 51015, "b9f3ef15"]]); + const storm = planTasks({ seed: 4242, tasks: 3, workloads: ["A1"], adapters: ["codex"], faults: ["K1"], triggers: ["time"], stormMinutes: 10 } as never); + expect(storm.map(task => [task.fault, task.fireAfterMs, task.nonce])).toEqual([["K13", 0, "84e8a446"], ["K13", 0, "59bc9f5c"], ["K13", 0, "d3a12313"]]); + }); + it("raises the K4 hold upper end with --k4-hold-max-seconds, up to the 600 s outage cap", () => { const command = parseCommand(["--faults", "K4", "--k4-hold-max-seconds", "600", "--seed", "5"]); if (command.mode !== "wave") throw new Error("expected a wave"); diff --git a/tests/restart-chaos/scheduler.ts b/tests/restart-chaos/scheduler.ts index 696c4799ead..fce5c82f9cc 100644 --- a/tests/restart-chaos/scheduler.ts +++ b/tests/restart-chaos/scheduler.ts @@ -80,7 +80,11 @@ function stormCellFaults(options: Pick, * Pairwise-friendly sampling: cycle a seeded shuffle of the full * workload × adapter × fault product so small waves still spread across cells. */ -export function planTasks(options: Pick): TaskPlan[] { +function pinnedFireAfterMs(faultAfterMs: number | undefined, drawn: number): number { + return faultAfterMs ?? drawn; +} + +export function planTasks(options: Pick): TaskPlan[] { const rng = createRng(options.seed); const triggers = options.triggers?.length ? options.triggers : ["time" as const]; for (const trigger of triggers) { @@ -110,7 +114,7 @@ export function planTasks(options: Pick