test(restart-chaos): TM1 control-plane stall on Daytona with --tm1-hold-seconds

Co-Authored-By: Claude <noreply@anthropic.com>

tests/restart-chaos/README.md

@@ -881,8 +881,26 @@ node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env local \
 # The aged wait: an approval left pending for a day across a restart.
 node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env local \
   --adapters codex --workloads G1 --faults TM3 --triggers approval --concurrency 2 --tasks 2 --seed 302
+# Staging's 100 s event-loop stall: TM1 on Daytona, stalled during workspace preparation or mid-tool.
+node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona --daytona-image $IMG --daytona-cpu 3 \
+  --adapters codex,claude,opencode --workloads A1,T1 --faults TM1 --tm1-hold-seconds 70-120 --storage s3 \
+  --concurrency 3 --tasks 6 --seed 303
 ```
 
+TM1 also runs on `--env daytona`. It stalls only the control plane, with no
+restart and no new deployment identity. The remote Runner keeps working while
+every controller lease of the stalled process lapses. This is staging's class L:
+a server stall of about 100 s at 08:19Z on Nov 10.
+
+`--tm1-hold-seconds <n>` or `<min>-<max>` (10–600 s) replaces TM1's default
+95–150 s range. It uses the same single draw, so seeds keep their cells.
+
+To stall a run while it is still `preparing`, use `--fault-after-seconds`, as in
+`--fault-after-seconds 10 --storage s3`. A stall while preparing needs a
+preparation that outlasts the fault time, such as a remote workspace restore.
+
+TM2 and TM3 age the wave database and freeze local runners, so they stay local.
+
 The fourth fault family. Every other wave is minutes long on a fresh instance, so nothing time-bound
 expires. A time fault makes deadlines lapse during a run or across a restart. No product code changes.
 

tests/restart-chaos/config.ts

@@ -134,13 +134,14 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC
       "rr-gap-seconds": { type: "string" },
       "rr-stop-on": { type: "string" },
       "k15-hold-seconds": { type: "string" },
+      "tm1-hold-seconds": { type: "string" },
     },
   });
   const keep = values.keep === true;
   const pollMs = integer(values["poll-ms"], "poll-ms", 3_000, 2_000, 5_000);
   const budgetCents = integer(values["budget-cents"], "budget-cents", 3_000, 1, 100_000);
   if (values.replay !== undefined) {
-    for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "daytona-reuse-lease", "daytona-warm", "daytona-inherit", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point", "crash-point-hold-ms", "upgrade-from", "workspace-repo", "storage", "storage-faults", "storage-fault-seconds", "storage-trigger", "native-trigger", "native-trigger-outage", "fault-after-seconds", "provider-fault-message", "provider-fault-delays", "db-pressure", "matrix", "server-from", "time-age-hours", "provider-fault-restart", "provider-fault-restart-after-seconds", "rr-gap-seconds", "rr-stop-on", "k15-hold-seconds"] as const) {
+    for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "daytona-reuse-lease", "daytona-warm", "daytona-inherit", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point", "crash-point-hold-ms", "upgrade-from", "workspace-repo", "storage", "storage-faults", "storage-fault-seconds", "storage-trigger", "native-trigger", "native-trigger-outage", "fault-after-seconds", "provider-fault-message", "provider-fault-delays", "db-pressure", "matrix", "server-from", "time-age-hours", "provider-fault-restart", "provider-fault-restart-after-seconds", "rr-gap-seconds", "rr-stop-on", "k15-hold-seconds", "tm1-hold-seconds"] as const) {
       if (values[key] !== undefined) throw new Error(`--replay reproduces the saved plan; do not combine it with --${key}`);
     }
     return { mode: "replay", ref: parseReplayRef(values.replay), keep,
@@ -242,7 +243,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC
   if (options.faults.includes("K15") && options.faults.some(isMultiServerFault)) {
     throw new Error("K15 (redeploy) does not combine with MS1-MS3: the second server would not share the emulated deploy identity");
   }
-  Object.assign(options, parseTimeFaultOptions(options, values["time-age-hours"]));
+  Object.assign(options, parseTimeFaultOptions(options, values["time-age-hours"], values["tm1-hold-seconds"]));
   Object.assign(options, parseRepeatedRestartOptions(options, values["rr-gap-seconds"], values["rr-stop-on"]));
   Object.assign(options, parseK15HoldSeconds(options, values["k15-hold-seconds"]));
   const upgradeFrom = values["upgrade-from"];

tests/restart-chaos/scheduler.ts

@@ -96,7 +96,7 @@ function pinnedFireAfterMs(faultAfterMs: number | undefined, drawn: number): num
   return faultAfterMs ?? drawn;
 }
 
-export function planTasks(options: Pick<WaveOptions, "seed" | "tasks" | "workloads" | "adapters" | "faults" | "triggers" | "stormMinutes" | "k4HoldMaxMs" | "k15HoldMs" | "faultAfterMs" | "providerFaults" | "rrGapMs">): TaskPlan[] {
+export function planTasks(options: Pick<WaveOptions, "seed" | "tasks" | "workloads" | "adapters" | "faults" | "triggers" | "stormMinutes" | "k4HoldMaxMs" | "k15HoldMs" | "tm1HoldMs" | "faultAfterMs" | "providerFaults" | "rrGapMs">): TaskPlan[] {
   const rng = createRng(options.seed);
   const triggers = options.triggers?.length ? options.triggers : ["time" as const];
   for (const trigger of triggers) {
@@ -127,7 +127,7 @@ export function planTasks(options: Pick<WaveOptions, "seed" | "tasks" | "workloa
       ...cellRest,
       ...(trigger === "time" ? {} : { trigger, triggerDelayMs: rng.int(...TRIGGER_DELAY_RANGE_MS[trigger]) }),
       fireAfterMs: cell.fault === "K13" ? 0 : pinnedFireAfterMs(options.faultAfterMs, rng.int(fireMin, fireMax)),
-      holdMs: cell.fault === "K4" ? rng.int(...k4HoldRange(options.k4HoldMaxMs)) : isTimeFault(cell.fault) ? timeFaultHoldMs(cell.fault, rng.int)
+      holdMs: cell.fault === "K4" ? rng.int(...k4HoldRange(options.k4HoldMaxMs)) : isTimeFault(cell.fault) ? timeFaultHoldMs(cell.fault, rng.int, options.tm1HoldMs)
         : isMultiServerFault(cell.fault) ? multiServerHoldMs(cell.fault, rng.int)
         : cell.fault === "K15" ? k15HoldMs(options.k15HoldMs, rng.int) : 0,
       // Drawn only for RR tasks, so the plans of existing seeds do not change.

tests/restart-chaos/time-faults-config.ts

@@ -18,29 +18,54 @@ export const TIME_FREEZE_HOLD_RANGE_MS: Record<"TM1" | "TM2", readonly [number,
 /** Default age: past a day, so hourly, daily and 1 h deadlines all lapse. `--time-age-hours` sets it (1–72). */
 export const DEFAULT_TIME_AGE_MS = DEFAULT_TIME_AGE_HOURS * 3_600_000;
 
-/** Seeded freeze length for a TM task; 0 for TM3 (its time passes through aging). Draws only for TM1/TM2. */
-export function timeFaultHoldMs(kind: TimeFaultKind, int: (min: number, max: number) => number): number {
-  return kind === "TM3" ? 0 : int(...TIME_FREEZE_HOLD_RANGE_MS[kind]);
+/**
+ * Seeded freeze length for a TM task; 0 for TM3 (its time passes through aging). Draws only for TM1/TM2.
+ * `tm1RangeMs` (`--tm1-hold-seconds`) replaces TM1's default range with the same single draw, so seeds keep their cells.
+ */
+export function timeFaultHoldMs(kind: TimeFaultKind, int: (min: number, max: number) => number,
+  tm1RangeMs?: readonly [number, number]): number {
+  if (kind === "TM3") return 0;
+  return int(...(kind === "TM1" && tm1RangeMs ? tm1RangeMs : TIME_FREEZE_HOLD_RANGE_MS[kind]));
 }
 
+/** `--tm1-hold-seconds` bounds: long enough to matter, within the controller-outage cap. */
+export const TM1_HOLD_SECONDS_RANGE: readonly [number, number] = [10, 600];
+
 const MIN_HOURS = 1;
 const MAX_HOURS = 72;
 
 export function parseTimeFaultOptions(options: Pick<WaveOptions, "env" | "faults" | "stormMinutes" | "crashPoint">,
-  rawAgeHours: string | undefined): Pick<WaveOptions, "timeAgeMs"> {
+  rawAgeHours: string | undefined, rawTm1HoldSeconds?: string): Pick<WaveOptions, "timeAgeMs" | "tm1HoldMs"> {
   const time = options.faults.filter(isTimeFault);
   const ages = time.some(kind => kind === "TM2" || kind === "TM3");
   if (rawAgeHours !== undefined && !ages) throw new Error("--time-age-hours sets the TM2/TM3 age; add TM2 or TM3 to --faults");
+  if (rawTm1HoldSeconds !== undefined && !time.includes("TM1")) throw new Error("--tm1-hold-seconds sets the TM1 stall; add TM1 to --faults");
   if (!time.length) return {};
-  // A freeze sends SIGSTOP to processes under the local server; a remote runner is not one of them.
-  if (options.env !== "local") throw new Error("Time faults (TM*) freeze and age a local instance; run them with --env local");
+  // TM1 stalls the control plane alone (SIGSTOP, then SIGCONT, no restart): on Daytona the remote Runner keeps
+  // working while the controller's leases lapse, as in staging's 100 s event-loop stall. TM2 and TM3 age the
+  // wave database and freeze local runners, so they stay local.
+  if (options.env !== "local" && time.some(kind => kind !== "TM1")) {
+    throw new Error("Time faults TM2 and TM3 freeze and age a local instance; run them with --env local (TM1 also runs on Daytona)");
+  }
   if (options.stormMinutes) throw new Error("Time faults (TM*) are planned per task; do not combine them with --storm-minutes");
   if (options.crashPoint) throw new Error("Time faults (TM*) do not combine with --crash-point");
   if (options.faults.includes("K14")) throw new Error("Time faults (TM*) do not combine with K14; age the upgraded build in its own wave");
-  if (!ages) return {};
-  if (rawAgeHours === undefined) return { timeAgeMs: DEFAULT_TIME_AGE_HOURS * 3_600_000 };
+  const tm1 = parseTm1HoldSeconds(rawTm1HoldSeconds);
+  if (!ages) return tm1;
+  if (rawAgeHours === undefined) return { ...tm1, timeAgeMs: DEFAULT_TIME_AGE_HOURS * 3_600_000 };
   if (!/^[0-9]+$/.test(rawAgeHours)) throw new Error("--time-age-hours must be a whole number of hours");
   const hours = Number(rawAgeHours);
   if (hours < MIN_HOURS || hours > MAX_HOURS) throw new Error(`--time-age-hours must be between ${MIN_HOURS} and ${MAX_HOURS}`);
-  return { timeAgeMs: hours * 3_600_000 };
+  return { ...tm1, timeAgeMs: hours * 3_600_000 };
+}
+
+function parseTm1HoldSeconds(raw: string | undefined): Pick<WaveOptions, "tm1HoldMs"> {
+  if (raw === undefined) return {};
+  const match = /^([0-9]+)(?:-([0-9]+))?$/.exec(raw.trim());
+  if (!match) throw new Error("--tm1-hold-seconds takes <seconds> or <min>-<max> seconds");
+  const min = Number(match[1]);
+  const max = match[2] === undefined ? min : Number(match[2]);
+  const [low, high] = TM1_HOLD_SECONDS_RANGE;
+  if (min < low || max > high || max < min) throw new Error(`--tm1-hold-seconds must be between ${low} and ${high} seconds, min not above max`);
+  return { tm1HoldMs: [min * 1000, max * 1000] };
 }

tests/restart-chaos/tm1-stall.test.ts

@@ -0,0 +1,40 @@
+import { describe, expect, it } from "vitest";
+import { parseCommand } from "./config.js";
+import { planTasks } from "./scheduler.js";
+import { TIME_FREEZE_HOLD_RANGE_MS, timeFaultHoldMs } from "./time-faults-config.js";
+
+const IMAGE = `ghcr.io/paperclipai/paperclip-daytona-runner@sha256:${"d".repeat(64)}`;
+
+describe("TM1 control-plane stall on Daytona (--tm1-hold-seconds)", () => {
+  it("stalls a Daytona wave's server without a restart, for the seeded range", () => {
+    const parsed = parseCommand(["--env", "daytona", "--daytona-image", IMAGE, "--faults", "TM1", "--tm1-hold-seconds", "70-120",
+      "--adapters", "codex,claude,opencode", "--workloads", "A1,T1", "--tasks", "6", "--seed", "21"]);
+    if (parsed.mode !== "wave") throw new Error("expected a wave");
+    expect(parsed.options.tm1HoldMs).toEqual([70_000, 120_000]);
+    const holds = planTasks(parsed.options).map(task => task.holdMs);
+    expect(holds.every(hold => hold >= 70_000 && hold <= 120_000)).toBe(true);
+  });
+
+  it("keeps the default TM1 range and every existing seed's cells without the option", () => {
+    const draws: Array<[number, number]> = [];
+    const int = (min: number, max: number) => { draws.push([min, max]); return min; };
+    expect(timeFaultHoldMs("TM1", int)).toBe(TIME_FREEZE_HOLD_RANGE_MS.TM1[0]);
+    expect(timeFaultHoldMs("TM1", int, [70_000, 120_000])).toBe(70_000);
+    expect(timeFaultHoldMs("TM3", int, [70_000, 120_000])).toBe(0);
+    expect(draws).toEqual([[...TIME_FREEZE_HOLD_RANGE_MS.TM1], [70_000, 120_000]]);
+    const a = parseCommand(["--faults", "TM1", "--workloads", "A1", "--tasks", "3", "--seed", "4"]);
+    const b = parseCommand(["--faults", "TM1", "--workloads", "A1", "--tasks", "3", "--seed", "4", "--tm1-hold-seconds", "100"]);
+    if (a.mode !== "wave" || b.mode !== "wave") throw new Error("expected waves");
+    const strip = (tasks: ReturnType<typeof planTasks>) => tasks.map(({ holdMs: _hold, ...rest }) => rest);
+    expect(strip(planTasks(b.options))).toEqual(strip(planTasks(a.options)));
+    expect(planTasks(b.options).every(task => task.holdMs === 100_000)).toBe(true);
+  });
+
+  it("keeps TM2 and TM3 local, and refuses a TM1 range without TM1 or out of bounds", () => {
+    expect(() => parseCommand(["--env", "daytona", "--daytona-image", IMAGE, "--faults", "TM1,TM3", "--workloads", "A1"])).toThrow(/TM2 and TM3 .*--env local/);
+    expect(() => parseCommand(["--faults", "K2", "--tm1-hold-seconds", "90", "--workloads", "A1"])).toThrow(/add TM1/);
+    expect(() => parseCommand(["--faults", "TM1", "--tm1-hold-seconds", "5", "--workloads", "A1"])).toThrow(/between 10 and 600/);
+    expect(() => parseCommand(["--faults", "TM1", "--tm1-hold-seconds", "120-70", "--workloads", "A1"])).toThrow(/min not above max/);
+    expect(() => parseCommand(["--replay", "w-1/t01", "--tm1-hold-seconds", "90"])).toThrow(/--replay reproduces the saved plan/);
+  });
+});

tests/restart-chaos/types.ts

@@ -71,6 +71,8 @@ export interface WaveOptions {
   k4HoldMaxMs?: number;
   /** `--k15-hold-seconds`: the seeded K15 outage range (redeploy.ts); absent restarts at once. */
   k15HoldMs?: [number, number];
+  /** `--tm1-hold-seconds`: the seeded TM1 stall range (time-faults-config.ts); absent uses 95–150 s. */
+  tm1HoldMs?: [number, number];
   faultAfterMs?: number;
   /**
    * Known product signatures this wave deliberately masks. "rc06" keeps the