diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index a76144c86f0..2e015e6a7c2 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -425,7 +425,7 @@ for a while after the new one was up. Plain K15 restarts on the same instance ho Pass is the usual grade: no Retry (R3), no stuck record (R6), and the oracle. Staging's terminal errors here were `ENOENT` on a runtime asset, `Agent directory operation is already active` after a 9.5-minute `controller_lease_held` wait. -### Staging class R cells (CR1–CR5, `class-r.ts`) +### Staging class R cells (CR1–CR6, `class-r.ts`) ```sh node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona --daytona-image $IMG --daytona-cpu 3 \ @@ -449,6 +449,7 @@ and the R1–R6 grading rules already exist. | CR3 | SIGSTOP of the server's Daytona plugin worker and its children. Every plugin RPC, including `environmentExecute`, stalls. There is no deploy. | `--cr3-stall-seconds `, 41–600. Default 45. | | CR4 | A K1 restart (SIGKILL, no hold). When the replacement is healthy and its server log shows the plugin activated (`plugin-loader: loadAll complete`), its Daytona plugin worker is stopped for the CR3 stall. A stall during activation is a different fault: the plugin's `initialize` call times out and the plugin stays in error. The run's reattachment reads stall while the run recovers: the saved state, the identity marker and the process check. | `--cr3-stall-seconds `. | | CR5 | The CR3 stall, fired 3–8 s after the task is assigned. It always uses the time trigger and ignores `--fault-after-seconds`. It covers the run's launch: the runnerd launch call and the identity read. It does not need to cross the T1 tool loop. | `--cr3-stall-seconds `. | +| CR6 | A K1 restart (SIGKILL, no hold). The replacement's Daytona plugin worker is stopped for the CR3 stall as soon as it forks, while the server activates the plugin at boot. The worker's `initialize` call times out (15 s). | `--cr3-stall-seconds `. | - CR2 kills only the processes under the run's `paperclip-runnerd` whose arguments match the adapter's provider command. For Claude (ACPX) that is `--output-format stream-json`. For OpenCode it is @@ -458,10 +459,10 @@ and the R1–R6 grading rules already exist. - The run id comes from the task's last sample before the fault. Each kill's output is saved to `daytona-runner/provider-kill--.json`. - Each CR task must pass `class_r_fault_applied`. The fault must not have failed, CR2 must have killed a - provider process, and the CR3, CR4 or CR5 stall must have ended. A cell whose fault never applied tested + provider process, and the CR3–CR6 stall must have ended. A cell whose fault never applied tested nothing. - Use T1, so that the fault lands while a tool runs, as it did on staging. CR5 is the exception: it lands at - launch, so T1's `fault_crossed_tool` check does not apply to it. Use `--concurrency 1` for CR3–CR5, + launch, so T1's `fault_crossed_tool` check does not apply to it. Use `--concurrency 1` for CR3–CR6, because the stall hits every task's plugin RPCs. **Sandbox capture.** Every Daytona wave, not only a CR wave, runs the class R probe (`CLASS_R_PROBE`) in diff --git a/tests/restart-chaos/class-r.test.ts b/tests/restart-chaos/class-r.test.ts index 76a766d79ad..d4f7feec498 100644 --- a/tests/restart-chaos/class-r.test.ts +++ b/tests/restart-chaos/class-r.test.ts @@ -1,7 +1,10 @@ -import { describe, expect, it } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { describe, expect, it, vi } from "vitest"; import { CR1_DEFAULT_HOLD_SECONDS, CR5_FIRE_WINDOW_MS, PROVIDER_KILL_SCRIPT, classRHoldMs, classRRedeploys, findDaytonaPluginWorker, killedPidsFromOutput, - parseClassROptions, providerKillCommand, runPluginStall, waitForPluginActivation, watchLabels, withClassRCheck, type ClassRRecord, + parseClassROptions, providerKillCommand, runPluginStall, waitForPluginActivation, watchLabels, withReplacementPluginStall, withClassRCheck, type ClassRRecord, } from "./class-r.js"; import { executeInWaveSandboxes } from "./daytona-runner-logs.js"; import { parseCommand } from "./config.js"; @@ -35,7 +38,7 @@ describe("class R options", () => { it("needs Daytona, its own fault for each option and no storm or crash point", () => { expect(() => parseClassROptions({ ...base, env: "local", faults: ["CR1"] }, {})).toThrow(/--env daytona/); expect(() => parseClassROptions({ ...base, faults: ["K15"] }, { cr1: "60" })).toThrow(/add CR1 to --faults/); - expect(() => parseClassROptions({ ...base, faults: ["K15"] }, { cr3: "60" })).toThrow(/add CR3, CR4 or CR5 to --faults/); + expect(() => parseClassROptions({ ...base, faults: ["K15"] }, { cr3: "60" })).toThrow(/add CR3, CR4, CR5 or CR6 to --faults/); expect(parseClassROptions({ ...base, faults: ["CR4", "CR5"] }, { cr3: "50" })).toEqual({ cr3StallMs: [50_000, 50_000] }); expect(() => parseClassROptions({ ...base, stormMinutes: 10, faults: ["CR1"] }, {})).toThrow(/storm/); expect(() => parseClassROptions({ ...base, faults: ["CR1", "MS1"] }, {})).toThrow(/MS1-MS3/); @@ -175,6 +178,26 @@ describe("CR3 stall", () => { readLog: () => lines.slice(0, ++reads).join("\n") }); expect(result).toBe(outcome); }); + it.each([ + ["CR6", "stops the worker while the plugin activates", ["starting", 'err: "Worker initialize failed: RPC call \\"initialize\\" timed out after 15000ms"'], 1], + ["CR4", "waits for the activation before it stops the worker", ["plugin-loader: loadAll complete"], 1], + ["K1", "leaves the plugin alone", ["plugin-loader: loadAll complete"], 0], + ] as const)("%s %s", async (kind, _what, log, frozen) => { + const root = mkdtempSync(path.join(tmpdir(), "class-r-replacement-")); + try { + const logPath = path.join(root, "server.log"); + writeFileSync(logPath, ["old boot: plugin-loader: loadAll complete", ...log].join("\n")); + const freeze = vi.fn(() => [11]); + const into: { pluginStall?: unknown } = {}; + const healthyAt = await withReplacementPluginStall(kind, { ...base, holdMs: 0, logPath, logOffset: "old boot: plugin-loader: loadAll complete".length, + activationTimeoutMs: 0, into: into as never, freezer: { freeze, thaw: () => {} } } as never, async () => 1234); + expect(healthyAt).toBe(1234); + expect(freeze).toHaveBeenCalledTimes(frozen); + if (frozen) expect(into.pluginStall).toMatchObject({ workerPid: 11, error: null }); + if (kind === "CR4") await expect(withReplacementPluginStall(kind, { ...base, holdMs: 0, logPath, logOffset: 10_000, + activationTimeoutMs: 0, into: {}, freezer: { freeze, thaw: () => {} } } as never, async () => 1)).rejects.toThrow(/activation timed_out/); + } finally { rmSync(root, { recursive: true, force: true }); } + }); it("fails when the worker could not be stopped or is missing", async () => { const missed = await runPluginStall({ ...base, freezer: { freeze: () => [], thaw: () => {} } }); expect(missed.error).toMatch(/could not stop/); @@ -236,6 +259,41 @@ describe("class_r_fault_applied", () => { expect(graded.verdict).toBe("fail"); expect(graded.checks.at(-1)!.detail).toContain("no running run"); }); + it("fails CR6 when the stopped worker had already finished initialize", async () => { + const root = mkdtempSync(path.join(tmpdir(), "class-r-cr6-evidence-")); + try { + const logPath = path.join(root, "server.log"); + writeFileSync(logPath, "worker process started and initialized"); + const into: { pluginStall?: { error: string | null } } = {}; + await expect(withReplacementPluginStall("CR6", { serverPid: 10, holdMs: 0, cancelled: () => false, record: () => {}, logPath, logOffset: 0, + into: into as never, table: () => [{ pid: 10, ppid: 1, command: "node server" }, { pid: 11, ppid: 10, command: "node /x/daytona/dist/worker.js" }], + sleep: async () => {}, freezer: { freeze: () => [11], thaw: () => {} } } as never, async () => 1)).rejects.toThrow(/no initialize timeout/); + expect(into.pluginStall?.error).toMatch(/after its initialize finished/); + } finally { rmSync(root, { recursive: true, force: true }); } + }); + it("cancels and joins the replacement stall when the replacement never becomes healthy", async () => { + const root = mkdtempSync(path.join(tmpdir(), "class-r-health-")); + try { + const logPath = path.join(root, "server.log"); + writeFileSync(logPath, "starting"); + const freeze = vi.fn(() => [11]); + const startedAt = Date.now(); + await expect(withReplacementPluginStall("CR4", { serverPid: 10, holdMs: 60_000, cancelled: () => false, record: () => {}, logPath, logOffset: 0, + activationTimeoutMs: 60_000, into: {}, sleep: (ms: number) => new Promise(resolve => setTimeout(resolve, Math.min(ms, 20))), + table: () => [], freezer: { freeze, thaw: () => {} } } as never, async () => { throw new Error("replacement never became healthy"); })) + .rejects.toThrow("replacement never became healthy"); + expect(Date.now() - startedAt).toBeLessThan(5_000); + expect(freeze).not.toHaveBeenCalled(); + } finally { rmSync(root, { recursive: true, force: true }); } + }); + it("fails CR5 when its stall started after the launch window", () => { + const stall = { workerPid: 11, command: "node", frozenPids: [11], frozenAt: 10_000, thawedAt: 55_000, error: null }; + const fault = (requestedAt: number) => [{ requestedBy: "t01", kind: "CR5", error: null, requestedAt, classR: classR({ kind: "CR5", pluginStall: stall }) }]; + expect(withClassRCheck(oracle, { taskId: "t01", fault: "CR5" }, fault(8_000)).verdict).toBe("pass"); + const late = withClassRCheck(oracle, { taskId: "t01", fault: "CR5" }, fault(-15_000)); + expect(late.verdict).toBe("fail"); + expect(late.checks.at(-1)!.detail).toContain("25000ms after its launch-window time"); + }); it("passes a CR3 task only when the stall ended", () => { const stall = { workerPid: 11, command: "node", frozenPids: [11], frozenAt: 1_000, thawedAt: 46_000, error: null }; expect(withClassRCheck(oracle, { taskId: "t01", fault: "CR3" }, [{ requestedBy: "t01", kind: "CR3", error: null, @@ -243,7 +301,7 @@ describe("class_r_fault_applied", () => { }); it.each(["CR4", "CR5"])("passes a %s task only when its stall ended", fault => { const stall = { workerPid: 11, command: "node", frozenPids: [11], frozenAt: 1_000, thawedAt: 46_000, error: null }; - const applied = withClassRCheck(oracle, { taskId: "t01", fault }, [{ requestedBy: "t01", kind: fault, error: null, + const applied = withClassRCheck(oracle, { taskId: "t01", fault }, [{ requestedBy: "t01", kind: fault, error: null, requestedAt: 1_000, classR: classR({ kind: fault as ClassRRecord["kind"], pluginStall: stall }) }]); expect(applied.verdict).toBe("pass"); expect(applied.checks.at(-1)!.detail).toContain("45000ms"); diff --git a/tests/restart-chaos/class-r.ts b/tests/restart-chaos/class-r.ts index 91938c04e2f..57d53ef95f1 100644 --- a/tests/restart-chaos/class-r.ts +++ b/tests/restart-chaos/class-r.ts @@ -16,6 +16,8 @@ * - CR4: a K1 restart (SIGKILL, no hold); as soon as the replacement is healthy its Daytona plugin worker is * stopped for the same stall, so the run's reattachment reads (saved state, identity marker, process check) * stall while the run recovers. + * - CR6: a K1 restart; the replacement's Daytona plugin worker is stopped as soon as it forks, while the server + * activates the plugin at boot, so its `initialize` call times out. * - CR5: the CR3 stall, fired a few seconds after the task is assigned, so it covers the run's launch: the * runnerd launch call and its identity read. The stall is not meant to cross the T1 tool loop. * @@ -32,7 +34,7 @@ import { readFileSync } from "node:fs"; import type { AdapterId, FaultKind, FaultReceipt, WaveOptions } from "./types.js"; import { ProcessFreezer, descendantPids, readProcessTable, type ProcessRow } from "./time-faults.js"; -export const CLASS_R_FAULT_KINDS = ["CR1", "CR2", "CR3", "CR4", "CR5"] as const; +export const CLASS_R_FAULT_KINDS = ["CR1", "CR2", "CR3", "CR4", "CR5", "CR6"] as const; export type ClassRFaultKind = (typeof CLASS_R_FAULT_KINDS)[number]; export function isClassRFault(kind: string): kind is ClassRFaultKind { @@ -44,7 +46,7 @@ export const CR1_DEFAULT_HOLD_SECONDS = 360; /** Staging's ninth run stalled one plugin RPC for more than 40 s. */ export const CR3_DEFAULT_STALL_SECONDS = 45; export const CR3_MIN_STALL_SECONDS = 41; -export const PLUGIN_STALL_FAULT_KINDS = ["CR3", "CR4", "CR5"] as const; +export const PLUGIN_STALL_FAULT_KINDS = ["CR3", "CR4", "CR5", "CR6"] as const; export function isPluginStallFault(kind: string): boolean { return (PLUGIN_STALL_FAULT_KINDS as readonly string[]).includes(kind); } @@ -87,12 +89,12 @@ export function parseClassROptions(options: Pick kind.startsWith("MS"))) throw new Error("Class R faults (CR1-CR5) do not combine with MS1-MS3"); + if (options.env !== "daytona") throw new Error("Class R faults (CR1-CR6) act on the remote Daytona sandbox; run them with --env daytona"); + if (options.stormMinutes) throw new Error("Class R faults (CR1-CR6) are planned per task; do not combine them with --storm-minutes"); + if (options.crashPoint) throw new Error("Class R faults (CR1-CR6) do not combine with --crash-point"); + if (options.faults.some(kind => kind.startsWith("MS"))) throw new Error("Class R faults (CR1-CR6) do not combine with MS1-MS3"); return { ...(kinds.includes("CR1") ? { cr1HoldMs: parseRange(values.cr1 ?? String(CR1_DEFAULT_HOLD_SECONDS), flags.CR1, 0, CLASS_R_MAX_HOLD_SECONDS) } : {}), ...(kinds.includes("CR2") ? { cr2HoldMs: parseRange(values.cr2 ?? "0", flags.CR2, 0, CLASS_R_MAX_HOLD_SECONDS) } : {}), @@ -251,20 +253,48 @@ export async function runPluginStall(input: { return stall; } -export async function stallAfterPluginActivation(input: { +export interface ReplacementPluginStallInput { serverPid: number; holdMs: number; cancelled: () => boolean; record: (event: string, data: Record) => void; - logPath: string; logOffset: number; into: { pluginStall?: PluginStallRecord }; activationTimeoutMs?: number; -}): Promise { - const activation = await waitForPluginActivation({ timeoutMs: input.activationTimeoutMs ?? 120_000, cancelled: input.cancelled, - readLog: () => { try { return readFileSync(input.logPath).subarray(input.logOffset).toString("utf8"); } catch { return ""; } } }); + logPath: string; logOffset: number; into: { pluginStall?: PluginStallRecord } | undefined; activationTimeoutMs?: number; +} + +export const CR6_INITIALIZE_TIMEOUT_EVIDENCE = /RPC call \\?"initialize\\?" timed out/; + +export async function stallReplacementPlugin(input: ReplacementPluginStallInput & { afterActivation: boolean }): Promise { + const readLog = () => { try { return readFileSync(input.logPath).subarray(input.logOffset).toString("utf8"); } catch { return ""; } }; + const activation = input.afterActivation + ? await waitForPluginActivation({ timeoutMs: input.activationTimeoutMs ?? 120_000, cancelled: input.cancelled, readLog }) + : "activated"; const stall = activation === "activated" ? await runPluginStall({ ...input, findWorkerWithinMs: 60_000 }) : { workerPid: null, command: null, frozenPids: [], frozenAt: null, thawedAt: null, error: `The replacement's plugin activation ${activation}; CR4 stalls only an activated plugin` }; - input.into.pluginStall = stall; + if (!input.afterActivation && stall.error === null && !CR6_INITIALIZE_TIMEOUT_EVIDENCE.test(readLog())) { + stall.error = "CR6 stopped the plugin worker after its initialize finished; no initialize timeout is in the replacement's log"; + } + if (input.into) input.into.pluginStall = stall; if (stall.error) throw new Error(stall.error); return stall; } +export async function withReplacementPluginStall(kind: FaultKind, input: ReplacementPluginStallInput, + waitForHealth: () => Promise): Promise { + if (kind !== "CR4" && kind !== "CR6") return waitForHealth(); + let healthFailed = false; + const stall = stallReplacementPlugin({ ...input, afterActivation: kind === "CR4", cancelled: () => healthFailed || input.cancelled() }) + .then(() => null, (error: unknown) => error); + let healthyAt: number; + try { + healthyAt = await waitForHealth(); + } catch (error) { + healthFailed = true; + await stall; + throw error; + } + const failure = await stall; + if (failure) throw failure; + return healthyAt; +} + export function findDaytonaPluginWorker(rows: ReadonlyArray<{ pid: number; ppid: number; command: string }>, serverPid: number, descendants: (root: number) => number[]): { pid: number; command: string } | null { const under = new Set(descendants(serverPid)); @@ -305,8 +335,11 @@ export interface ClassRRecord { * CR2 and CR3 only test something when their fault applied: a provider was killed, or the plugin worker was * stopped for the whole stall. A CR task without that fails `class_r_fault_applied`. */ +export const CR5_LATEST_STALL_START_MS = 5_000; + export function withClassRCheck }>( - oracle: T, plan: { taskId: string; fault: string }, faults: ReadonlyArray<{ requestedBy: string; kind: string; classR?: ClassRRecord; error: string | null }>): T { + oracle: T, plan: { taskId: string; fault: string }, + faults: ReadonlyArray<{ requestedBy: string; kind: string; classR?: ClassRRecord; error: string | null; requestedAt?: number }>): T { if (!isClassRFault(plan.fault)) return oracle; const own = faults.filter(fault => fault.requestedBy === plan.taskId && fault.kind === plan.fault); let ok = own.length > 0 && own.every(fault => fault.error === null); @@ -321,6 +354,11 @@ export function withClassRCheck 0 && stalls.every(stall => stall?.frozenAt != null && stall.thawedAt != null); detail += `; stalled ${stalls.map(stall => stall?.frozenAt != null && stall.thawedAt != null ? `${stall.thawedAt - stall.frozenAt}ms` : `no(${stall?.error ?? "?"})`).join(",")}`; } + if (plan.fault === "CR5") { + const delays = own.map(fault => (fault.classR?.pluginStall?.frozenAt ?? Infinity) - (fault.requestedAt ?? -Infinity)); + ok &&= delays.every(delay => delay <= CR5_LATEST_STALL_START_MS); + detail += `; started ${delays.map(delay => Number.isFinite(delay) ? `${delay}ms` : "unknown").join(",")} after its launch-window time`; + } const checks = [...oracle.checks, { name: "class_r_fault_applied", ok, detail }]; return { ...oracle, checks, verdict: checks.every(check => check.ok) ? "pass" : "fail" }; } diff --git a/tests/restart-chaos/supervisor.ts b/tests/restart-chaos/supervisor.ts index 979343168c4..7cd0610fef4 100644 --- a/tests/restart-chaos/supervisor.ts +++ b/tests/restart-chaos/supervisor.ts @@ -42,7 +42,7 @@ import { isMultiServerFault, runMultiServerFault, type MultiServerKind, type Mul import { generationHome, nextRedeployIdentity, redeployEnvironment, redeployIdentity, redeployNodeArgs, redeployReceipt, type RedeployIdentity } from "./redeploy.js"; import { startEdgeProxy, type EdgeProxy } from "./edge-proxy.js"; import { ephemeralServerEnvironment } from "./ephemeral-server.js"; -import { CLASS_R_PRE_FAULT_DEADLINE_MS, classRRedeploys, isClassRFault, runPluginStall, stallAfterPluginActivation, type ClassRHooks } from "./class-r.js"; +import { CLASS_R_PRE_FAULT_DEADLINE_MS, classRRedeploys, isClassRFault, runPluginStall, withReplacementPluginStall, type ClassRHooks } from "./class-r.js"; /** First boot applies every migration; a loaded host can need longer (RESTART_CHAOS_HEALTH_TIMEOUT_MS). */ const HEALTH_TIMEOUT_MS = (() => { @@ -57,8 +57,8 @@ export const FAULT_SIGNAL: Record = { K1: "SI // MS1 leaves the primary alone; MS2 freezes it; MS3 stops it gracefully (multi-server.ts). MS1: "none", MS2: "SIGSTOP", MS3: "SIGTERM", // CR1 stops the server like K15; CR2 signals only the sandbox's provider (SIGTERM to the server with a hold); - // CR3 and CR5 stop the Daytona plugin worker; CR4 kills the server, then stops the replacement's (class-r.ts). - CR1: "SIGTERM", CR2: "none", CR3: "SIGSTOP", CR4: "SIGKILL", CR5: "SIGSTOP" }; + // CR3 and CR5 stop the Daytona plugin worker; CR4 and CR6 kill the server, then stop the replacement's (class-r.ts). + CR1: "SIGTERM", CR2: "none", CR3: "SIGSTOP", CR4: "SIGKILL", CR5: "SIGSTOP", CR6: "SIGKILL" }; /** The checkout's commit, for the K14 receipt. */ function checkoutSha(root: string): string | null { @@ -513,7 +513,7 @@ export class ChaosSupervisor { receipt.classR = { kind, redeploy: redeploys, captures: [] }; if (kind === "CR2" && holdMs > 0) receipt.signal = "SIGTERM"; // The sandbox just before the fault, while the controller still holds it; the fault starts after it. - receipt.classR.captures.push(await this.classR.capture(`${receipt.faultId}-before`, CLASS_R_PRE_FAULT_DEADLINE_MS)); + if (kind !== "CR5") receipt.classR.captures.push(await this.classR.capture(`${receipt.faultId}-before`, CLASS_R_PRE_FAULT_DEADLINE_MS)); receipt.startedAt = Date.now(); } if (kind === "TM1" || kind === "TM2") await this.freeze(kind, previous, holdMs, receipt); @@ -536,7 +536,7 @@ export class ChaosSupervisor { else outage = await replaceController({ requestId: `chaos-${receipt.faultId}-${requestedBy}`, previousPid: previous.pid, - outage: { signal: receipt.signal === "SIGTERM" ? "SIGTERM" : "SIGKILL", holdMs: kind === "CR4" ? 0 : holdMs }, + outage: { signal: receipt.signal === "SIGTERM" ? "SIGTERM" : "SIGKILL", holdMs: kind === "CR4" || kind === "CR6" ? 0 : holdMs }, // A K4 hold may outlast the Runner's 7-minute reconnect grace on // purpose (--k4-hold-max-seconds); the replacement still gets the // normal startup allowance after the hold. @@ -585,9 +585,9 @@ export class ChaosSupervisor { receipt.repeatedRestart.firstStopPhase = await this.waitForCrashLoopPhase(replacement, receipt.repeatedRestart.stopOn, receipt.phases); return; } - receipt.healthyAt = await this.waitForHealth(replacement, receipt.phases); - if (kind === "CR4") await stallAfterPluginActivation({ into: receipt.classR!, serverPid: replacement.pid!, holdMs, cancelled: () => this.shuttingDown, - logPath: this.paths.serverLogPath, logOffset: receipt.serverLogOffsetAtStart, record: (event, data) => this.timeline.add(event, { faultId: receipt.faultId, ...data }) }); + receipt.healthyAt = await withReplacementPluginStall(kind, { into: receipt.classR, serverPid: replacement.pid!, holdMs, cancelled: () => this.shuttingDown, + logPath: this.paths.serverLogPath, logOffset: receipt.serverLogOffsetAtStart, record: (event, data) => this.timeline.add(event, { faultId: receipt.faultId, ...data }) }, + () => this.waitForHealth(replacement, receipt.phases)); }, cancelled: () => this.shuttingDown, record: async progress => { outage = progress; }, diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index 441bb8d85f9..9a342b9db7a 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -12,10 +12,11 @@ import type { TimeFaultRecord } from "./time-faults.js"; * TM1–TM3 (time faults, time-faults.ts): freeze the server, sleep with aged rows, or an aged outage. * RR1–RR3 (repeated restarts, repeated-restarts.ts): a K1, then more restarts while the replacement recovers. * MS1–MS3 (two servers, one database, multi-server.ts): a second server beside the first. - * CR1–CR5 (staging class R, class-r.ts): a long-held redeploy, a provider-only kill in the sandbox, a plugin RPC stall - * mid-tool (CR3), right after a K1 restart while the run reattaches (CR4), or while the run launches (CR5). + * CR1–CR6 (staging class R, class-r.ts): a long-held redeploy, a provider-only kill in the sandbox, a plugin RPC stall + * mid-tool (CR3), right after a K1 restart while the run reattaches (CR4), while the run launches (CR5), or while the + * replacement activates the plugin at boot (CR6). */ -export const FAULT_KINDS = ["K1", "K2", "K4", "K6", "K14", "K15", "TM1", "TM2", "TM3", "RR1", "RR2", "RR3", "MS1", "MS2", "MS3", "CR1", "CR2", "CR3", "CR4", "CR5"] as const; +export const FAULT_KINDS = ["K1", "K2", "K4", "K6", "K14", "K15", "TM1", "TM2", "TM3", "RR1", "RR2", "RR3", "MS1", "MS2", "MS3", "CR1", "CR2", "CR3", "CR4", "CR5", "CR6"] as const; export type FaultKind = (typeof FAULT_KINDS)[number]; /** The shortest seeded K4 outage (scheduler.ts K4_HOLD_RANGE_MS), also the K4 hold after a provider fault. */ export const K4_MIN_HOLD_MS = 120_000; @@ -104,7 +105,7 @@ export interface WaveOptions { cr1HoldMs?: [number, number]; /** `--cr2-hold-seconds`: a K15 outage around CR2's provider kill; absent or 0 keeps the controller up. */ cr2HoldMs?: [number, number]; - /** `--cr3-stall-seconds`: how long CR3, CR4 and CR5 stop the Daytona plugin worker; absent means 45 s. */ + /** `--cr3-stall-seconds`: how long CR3–CR6 stop the Daytona plugin worker; absent means 45 s. */ cr3StallMs?: [number, number]; /** `--tm1-hold-seconds`: the seeded TM1 stall range (time-faults-config.ts); absent uses 95–150 s. */ tm1HoldMs?: [number, number]; @@ -342,7 +343,7 @@ export interface FaultReceipt { repeatedRestart?: RepeatedRestartRecord; /** MS1–MS3: the second server and what happened to the first. */ multiServer?: import("./multi-server.js").MultiServerRecord; - /** CR1–CR5: the provider kill or plugin stall and the sandbox captures taken around the fault (class-r.ts). */ + /** CR1–CR6: the provider kill or plugin stall and the sandbox captures taken around the fault (class-r.ts). */ classR?: import("./class-r.js").ClassRRecord; serverLogOffsetAtStart: number; serverLogOffsetAtReady: number | null;