diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 97c2ddf7b32..44bdfc95349 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -881,8 +881,26 @@ node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env local \ # The aged wait: an approval left pending for a day across a restart. node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env local \ --adapters codex --workloads G1 --faults TM3 --triggers approval --concurrency 2 --tasks 2 --seed 302 +# Staging's 100 s event-loop stall: TM1 on Daytona, stalled during workspace preparation or mid-tool. +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona --daytona-image $IMG --daytona-cpu 3 \ + --adapters codex,claude,opencode --workloads A1,T1 --faults TM1 --tm1-hold-seconds 70-120 --storage s3 \ + --concurrency 3 --tasks 6 --seed 303 ``` +TM1 also runs on `--env daytona`. It stalls only the control plane, with no +restart and no new deployment identity. The remote Runner keeps working while +every controller lease of the stalled process lapses. This is staging's class L: +a server stall of about 100 s at 08:19Z on Nov 10. + +`--tm1-hold-seconds ` or `-` (10–600 s) replaces TM1's default +95–150 s range. It uses the same single draw, so seeds keep their cells. + +To stall a run while it is still `preparing`, use `--fault-after-seconds`, as in +`--fault-after-seconds 10 --storage s3`. A stall while preparing needs a +preparation that outlasts the fault time, such as a remote workspace restore. + +TM2 and TM3 age the wave database and freeze local runners, so they stay local. + The fourth fault family. Every other wave is minutes long on a fresh instance, so nothing time-bound expires. A time fault makes deadlines lapse during a run or across a restart. No product code changes. diff --git a/tests/restart-chaos/config.ts b/tests/restart-chaos/config.ts index b21e4960fc2..6e9ab880600 100644 --- a/tests/restart-chaos/config.ts +++ b/tests/restart-chaos/config.ts @@ -134,13 +134,14 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC "rr-gap-seconds": { type: "string" }, "rr-stop-on": { type: "string" }, "k15-hold-seconds": { type: "string" }, + "tm1-hold-seconds": { type: "string" }, }, }); const keep = values.keep === true; const pollMs = integer(values["poll-ms"], "poll-ms", 3_000, 2_000, 5_000); const budgetCents = integer(values["budget-cents"], "budget-cents", 3_000, 1, 100_000); if (values.replay !== undefined) { - for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "daytona-reuse-lease", "daytona-warm", "daytona-inherit", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point", "crash-point-hold-ms", "upgrade-from", "workspace-repo", "storage", "storage-faults", "storage-fault-seconds", "storage-trigger", "native-trigger", "native-trigger-outage", "fault-after-seconds", "provider-fault-message", "provider-fault-delays", "db-pressure", "matrix", "server-from", "time-age-hours", "provider-fault-restart", "provider-fault-restart-after-seconds", "rr-gap-seconds", "rr-stop-on", "k15-hold-seconds"] as const) { + for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "daytona-reuse-lease", "daytona-warm", "daytona-inherit", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point", "crash-point-hold-ms", "upgrade-from", "workspace-repo", "storage", "storage-faults", "storage-fault-seconds", "storage-trigger", "native-trigger", "native-trigger-outage", "fault-after-seconds", "provider-fault-message", "provider-fault-delays", "db-pressure", "matrix", "server-from", "time-age-hours", "provider-fault-restart", "provider-fault-restart-after-seconds", "rr-gap-seconds", "rr-stop-on", "k15-hold-seconds", "tm1-hold-seconds"] as const) { if (values[key] !== undefined) throw new Error(`--replay reproduces the saved plan; do not combine it with --${key}`); } return { mode: "replay", ref: parseReplayRef(values.replay), keep, @@ -242,7 +243,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC if (options.faults.includes("K15") && options.faults.some(isMultiServerFault)) { throw new Error("K15 (redeploy) does not combine with MS1-MS3: the second server would not share the emulated deploy identity"); } - Object.assign(options, parseTimeFaultOptions(options, values["time-age-hours"])); + Object.assign(options, parseTimeFaultOptions(options, values["time-age-hours"], values["tm1-hold-seconds"])); Object.assign(options, parseRepeatedRestartOptions(options, values["rr-gap-seconds"], values["rr-stop-on"])); Object.assign(options, parseK15HoldSeconds(options, values["k15-hold-seconds"])); const upgradeFrom = values["upgrade-from"]; diff --git a/tests/restart-chaos/scheduler.ts b/tests/restart-chaos/scheduler.ts index 8ebc2afea84..3ba1b1f4796 100644 --- a/tests/restart-chaos/scheduler.ts +++ b/tests/restart-chaos/scheduler.ts @@ -96,7 +96,7 @@ function pinnedFireAfterMs(faultAfterMs: number | undefined, drawn: number): num return faultAfterMs ?? drawn; } -export function planTasks(options: Pick): TaskPlan[] { +export function planTasks(options: Pick): TaskPlan[] { const rng = createRng(options.seed); const triggers = options.triggers?.length ? options.triggers : ["time" as const]; for (const trigger of triggers) { @@ -127,7 +127,7 @@ export function planTasks(options: Pick number): number { - return kind === "TM3" ? 0 : int(...TIME_FREEZE_HOLD_RANGE_MS[kind]); +/** + * Seeded freeze length for a TM task; 0 for TM3 (its time passes through aging). Draws only for TM1/TM2. + * `tm1RangeMs` (`--tm1-hold-seconds`) replaces TM1's default range with the same single draw, so seeds keep their cells. + */ +export function timeFaultHoldMs(kind: TimeFaultKind, int: (min: number, max: number) => number, + tm1RangeMs?: readonly [number, number]): number { + if (kind === "TM3") return 0; + return int(...(kind === "TM1" && tm1RangeMs ? tm1RangeMs : TIME_FREEZE_HOLD_RANGE_MS[kind])); } +/** `--tm1-hold-seconds` bounds: long enough to matter, within the controller-outage cap. */ +export const TM1_HOLD_SECONDS_RANGE: readonly [number, number] = [10, 600]; + const MIN_HOURS = 1; const MAX_HOURS = 72; export function parseTimeFaultOptions(options: Pick, - rawAgeHours: string | undefined): Pick { + rawAgeHours: string | undefined, rawTm1HoldSeconds?: string): Pick { const time = options.faults.filter(isTimeFault); const ages = time.some(kind => kind === "TM2" || kind === "TM3"); if (rawAgeHours !== undefined && !ages) throw new Error("--time-age-hours sets the TM2/TM3 age; add TM2 or TM3 to --faults"); + if (rawTm1HoldSeconds !== undefined && !time.includes("TM1")) throw new Error("--tm1-hold-seconds sets the TM1 stall; add TM1 to --faults"); if (!time.length) return {}; - // A freeze sends SIGSTOP to processes under the local server; a remote runner is not one of them. - if (options.env !== "local") throw new Error("Time faults (TM*) freeze and age a local instance; run them with --env local"); + // TM1 stalls the control plane alone (SIGSTOP, then SIGCONT, no restart): on Daytona the remote Runner keeps + // working while the controller's leases lapse, as in staging's 100 s event-loop stall. TM2 and TM3 age the + // wave database and freeze local runners, so they stay local. + if (options.env !== "local" && time.some(kind => kind !== "TM1")) { + throw new Error("Time faults TM2 and TM3 freeze and age a local instance; run them with --env local (TM1 also runs on Daytona)"); + } if (options.stormMinutes) throw new Error("Time faults (TM*) are planned per task; do not combine them with --storm-minutes"); if (options.crashPoint) throw new Error("Time faults (TM*) do not combine with --crash-point"); if (options.faults.includes("K14")) throw new Error("Time faults (TM*) do not combine with K14; age the upgraded build in its own wave"); - if (!ages) return {}; - if (rawAgeHours === undefined) return { timeAgeMs: DEFAULT_TIME_AGE_HOURS * 3_600_000 }; + const tm1 = parseTm1HoldSeconds(rawTm1HoldSeconds); + if (!ages) return tm1; + if (rawAgeHours === undefined) return { ...tm1, timeAgeMs: DEFAULT_TIME_AGE_HOURS * 3_600_000 }; if (!/^[0-9]+$/.test(rawAgeHours)) throw new Error("--time-age-hours must be a whole number of hours"); const hours = Number(rawAgeHours); if (hours < MIN_HOURS || hours > MAX_HOURS) throw new Error(`--time-age-hours must be between ${MIN_HOURS} and ${MAX_HOURS}`); - return { timeAgeMs: hours * 3_600_000 }; + return { ...tm1, timeAgeMs: hours * 3_600_000 }; +} + +function parseTm1HoldSeconds(raw: string | undefined): Pick { + if (raw === undefined) return {}; + const match = /^([0-9]+)(?:-([0-9]+))?$/.exec(raw.trim()); + if (!match) throw new Error("--tm1-hold-seconds takes or - seconds"); + const min = Number(match[1]); + const max = match[2] === undefined ? min : Number(match[2]); + const [low, high] = TM1_HOLD_SECONDS_RANGE; + if (min < low || max > high || max < min) throw new Error(`--tm1-hold-seconds must be between ${low} and ${high} seconds, min not above max`); + return { tm1HoldMs: [min * 1000, max * 1000] }; } diff --git a/tests/restart-chaos/tm1-stall.test.ts b/tests/restart-chaos/tm1-stall.test.ts new file mode 100644 index 00000000000..f71536db82f --- /dev/null +++ b/tests/restart-chaos/tm1-stall.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; +import { parseCommand } from "./config.js"; +import { planTasks } from "./scheduler.js"; +import { TIME_FREEZE_HOLD_RANGE_MS, timeFaultHoldMs } from "./time-faults-config.js"; + +const IMAGE = `ghcr.io/paperclipai/paperclip-daytona-runner@sha256:${"d".repeat(64)}`; + +describe("TM1 control-plane stall on Daytona (--tm1-hold-seconds)", () => { + it("stalls a Daytona wave's server without a restart, for the seeded range", () => { + const parsed = parseCommand(["--env", "daytona", "--daytona-image", IMAGE, "--faults", "TM1", "--tm1-hold-seconds", "70-120", + "--adapters", "codex,claude,opencode", "--workloads", "A1,T1", "--tasks", "6", "--seed", "21"]); + if (parsed.mode !== "wave") throw new Error("expected a wave"); + expect(parsed.options.tm1HoldMs).toEqual([70_000, 120_000]); + const holds = planTasks(parsed.options).map(task => task.holdMs); + expect(holds.every(hold => hold >= 70_000 && hold <= 120_000)).toBe(true); + }); + + it("keeps the default TM1 range and every existing seed's cells without the option", () => { + const draws: Array<[number, number]> = []; + const int = (min: number, max: number) => { draws.push([min, max]); return min; }; + expect(timeFaultHoldMs("TM1", int)).toBe(TIME_FREEZE_HOLD_RANGE_MS.TM1[0]); + expect(timeFaultHoldMs("TM1", int, [70_000, 120_000])).toBe(70_000); + expect(timeFaultHoldMs("TM3", int, [70_000, 120_000])).toBe(0); + expect(draws).toEqual([[...TIME_FREEZE_HOLD_RANGE_MS.TM1], [70_000, 120_000]]); + const a = parseCommand(["--faults", "TM1", "--workloads", "A1", "--tasks", "3", "--seed", "4"]); + const b = parseCommand(["--faults", "TM1", "--workloads", "A1", "--tasks", "3", "--seed", "4", "--tm1-hold-seconds", "100"]); + if (a.mode !== "wave" || b.mode !== "wave") throw new Error("expected waves"); + const strip = (tasks: ReturnType) => tasks.map(({ holdMs: _hold, ...rest }) => rest); + expect(strip(planTasks(b.options))).toEqual(strip(planTasks(a.options))); + expect(planTasks(b.options).every(task => task.holdMs === 100_000)).toBe(true); + }); + + it("keeps TM2 and TM3 local, and refuses a TM1 range without TM1 or out of bounds", () => { + expect(() => parseCommand(["--env", "daytona", "--daytona-image", IMAGE, "--faults", "TM1,TM3", "--workloads", "A1"])).toThrow(/TM2 and TM3 .*--env local/); + expect(() => parseCommand(["--faults", "K2", "--tm1-hold-seconds", "90", "--workloads", "A1"])).toThrow(/add TM1/); + expect(() => parseCommand(["--faults", "TM1", "--tm1-hold-seconds", "5", "--workloads", "A1"])).toThrow(/between 10 and 600/); + expect(() => parseCommand(["--faults", "TM1", "--tm1-hold-seconds", "120-70", "--workloads", "A1"])).toThrow(/min not above max/); + expect(() => parseCommand(["--replay", "w-1/t01", "--tm1-hold-seconds", "90"])).toThrow(/--replay reproduces the saved plan/); + }); +}); diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index aa3ab29e41c..9f553df1f74 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -71,6 +71,8 @@ export interface WaveOptions { k4HoldMaxMs?: number; /** `--k15-hold-seconds`: the seeded K15 outage range (redeploy.ts); absent restarts at once. */ k15HoldMs?: [number, number]; + /** `--tm1-hold-seconds`: the seeded TM1 stall range (time-faults-config.ts); absent uses 95–150 s. */ + tm1HoldMs?: [number, number]; faultAfterMs?: number; /** * Known product signatures this wave deliberately masks. "rc06" keeps the