test(restart-chaos): validate TTFT driver environment options; never fault on a failed runs read

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/restart-chaos/seed.test.ts

@@ -3,7 +3,7 @@ import os from "node:os";
 import path from "node:path";
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import type { ChaosApi } from "./api.js";
-import { seedChaosLab } from "./seed.js";
+import { inheritingEnvironmentConfig, seedChaosLab } from "./seed.js";
 import { Timeline } from "./timeline.js";
 
 type Call = { method: string; route: string; body?: unknown };
@@ -104,3 +104,11 @@ describe("seedChaosLab RC-06 mask", () => {
     expect(timeline.entries.some(entry => entry.kind === "seed_mask_rc06")).toBe(false);
   });
 });
+
+describe("inheritingEnvironmentConfig", () => {
+  it("drops the lifecycle and lease pins and keeps everything else", () => {
+    const fixture = { provider: "daytona", image: "img", runnerLifecycleMode: "per_turn", reuseLease: false };
+    expect(inheritingEnvironmentConfig(fixture)).toEqual({ provider: "daytona", image: "img" });
+    expect(fixture).toHaveProperty("runnerLifecycleMode", "per_turn");
+  });
+});

tests/restart-chaos/seed.ts

@@ -59,6 +59,12 @@ export function chaosDaytonaShape(cpu: number) {
   return { cpu, memory: 4, disk: 10 };
 }
 
+/** A user's default environment config: the lifecycle and lease reuse are left to the agent. */
+export function inheritingEnvironmentConfig(config: Record<string, unknown>): Record<string, unknown> {
+  const { runnerLifecycleMode: _lifecycle, reuseLease: _reuse, ...rest } = config;
+  return rest;
+}
+
 async function createDaytonaEnvironment(input: { api: ChaosApi; companyId: string; image: string; cpu: number; reuseLease: boolean; waveId: string;
   secretRefs: SecretReferenceMap; timeline: Timeline; warm?: boolean; inherit?: boolean }): Promise<string> {
   // The warm fixture keeps a reusable sandbox and a warm runner between turns.
@@ -69,13 +75,9 @@ async function createDaytonaEnvironment(input: { api: ChaosApi; companyId: strin
   const payload = fixture.buildEnvironment({ secretRefs: input.secretRefs, daytonaImage: input.image, executionId: input.waveId });
   const shape = chaosDaytonaShape(input.cpu);
   // A user's default environment: the agent decides the lifecycle, and nothing pins lease reuse.
-  if (input.inherit) {
-    const config = payload.config as Record<string, unknown>;
-    delete config.runnerLifecycleMode;
-    delete config.reuseLease;
-  }
+  const baseConfig = input.inherit ? inheritingEnvironmentConfig(payload.config as Record<string, unknown>) : payload.config as Record<string, unknown>;
   const environment = await input.api.post<{ id: string; driver: string }>(`/api/companies/${input.companyId}/environments`,
-    { ...payload, config: { ...(payload.config as Record<string, unknown>), ...shape, ...(input.reuseLease ? { reuseLease: true } : {}) } });
+    { ...payload, config: { ...baseConfig, ...shape, ...(input.reuseLease ? { reuseLease: true } : {}) } });
   input.timeline.add("seed_environment", { environmentId: environment.id, driver: environment.driver, image: input.image, ...shape,
     reuseLease: input.reuseLease });
   return environment.id;

tests/restart-chaos/ttft-baseline-stats.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from "vitest";
 import {
-  baselineRows, distribution, openShellExecutions, percentile, rankPreTurnSteps, recoverySamples, runMetrics, runWarmth,
+  baselineRows, daytonaBaselineHolders, distribution, parseBaselineEnvironment, openShellExecutions, percentile, rankPreTurnSteps, recoverySamples, runMetrics, runWarmth,
   type BaselineEvent, type BaselineRun,
 } from "./ttft-baseline-stats.js";
 
@@ -77,6 +77,31 @@ describe("ttft baseline statistics", () => {
       ttft: { n: 2, p50: 6_000, p95: 12_000 } });
   });
 
+  it("leaves post-restart and verification runs out of the baseline rows", () => {
+    const rows = baselineRows([coldRun("a"), run("p", "postfault", [span("task.time_to_first_token", 0, 20_000)]),
+      run("v", "verify", [span("task.time_to_first_token", 0, 30_000)])]);
+    expect(rows).toHaveLength(1);
+    expect(rows[0]).toMatchObject({ warmth: "cold", ttft: { n: 1, p50: 6_000 } });
+  });
+
+  it("counts held Daytona sandboxes per concurrent task, or per sampled task when they stay warm", () => {
+    const phases = new Set(["coldwarm", "midturn", "restart"]);
+    expect(daytonaBaselineHolders({ adapters: 3, samples: 5, phases, restarts: 2, retainsSandboxes: false })).toBe(3);
+    expect(daytonaBaselineHolders({ adapters: 3, samples: 5, phases, restarts: 2, retainsSandboxes: true })).toBe(60);
+    expect(daytonaBaselineHolders({ adapters: 2, samples: 0, phases: new Set(["postfault"]), restarts: 0, retainsSandboxes: true })).toBe(2);
+  });
+
+  it("rejects contradictory or out-of-range environment options", () => {
+    const base = { env: "daytona", image: "img", cpu: "4", warm: false, inherit: false };
+    expect(parseBaselineEnvironment(base)).toEqual({ env: "daytona", cpu: 4 });
+    expect(parseBaselineEnvironment({ ...base, env: "local", image: undefined })).toMatchObject({ env: "local" });
+    expect(() => parseBaselineEnvironment({ ...base, env: "dayton" })).toThrow(/local or daytona/);
+    expect(() => parseBaselineEnvironment({ ...base, image: undefined })).toThrow(/needs --daytona-image/);
+    for (const cpu of ["0", "5", "abc", "2.5"]) expect(() => parseBaselineEnvironment({ ...base, cpu })).toThrow(/1 to 4/);
+    expect(() => parseBaselineEnvironment({ ...base, warm: true, inherit: true })).toThrow(/cannot be combined/);
+    expect(() => parseBaselineEnvironment({ ...base, env: "local", image: undefined, warm: true })).toThrow(/only to --env daytona/);
+  });
+
   it("ranks pre-turn steps by p95 without cumulative wrappers", () => {
     const ranked = rankPreTurnSteps([coldRun("a"), coldRun("b", 2)]);
     expect(ranked[0]).toMatchObject({ step: "runner.process.launch", n: 2, p95: 3_000 });

tests/restart-chaos/ttft-baseline-stats.ts

@@ -235,3 +235,31 @@ export function openShellExecutions(events: readonly BaselineEvent[], marker?: s
   }
   return [...open.entries()].map(([executionId, entry]) => ({ executionId, matched: entry.matched, unlabelled: !entry.labelled }));
 }
+
+/** Daytona sandboxes a TTFT baseline can hold at once, for the vCPU budget check. */
+export function daytonaBaselineHolders(input: { adapters: number; samples: number; phases: ReadonlySet<string>; restarts: number;
+  retainsSandboxes: boolean }): number {
+  // The default fixture releases each sandbox after its turn, so only the concurrent tasks hold one.
+  if (!input.retainsSandboxes) return input.adapters;
+  // A warm sandbox outlives its task for the idle window and no other task reuses it.
+  const perTaskPhases = ["coldwarm", "midturn", "verify"].filter(phase => input.phases.has(phase)).length
+    + (input.phases.has("restart") ? Math.max(1, input.restarts) : 0);
+  return input.adapters * Math.max(1, input.samples) * Math.max(1, perTaskPhases);
+}
+
+/** Validates the TTFT driver's environment options; returns the environment. */
+export function parseBaselineEnvironment(input: { env: string | undefined; image: string | undefined; cpu: string | undefined;
+  warm: boolean; inherit: boolean }): { env: "local" | "daytona"; cpu: number } {
+  const env = input.env ?? "local";
+  if (env !== "local" && env !== "daytona") throw new Error(`--env must be local or daytona, got ${env}`);
+  const cpu = Number(input.cpu ?? "4");
+  if (env === "local") {
+    if (input.warm || input.inherit || input.image) throw new Error("--daytona-* options apply only to --env daytona");
+    return { env, cpu };
+  }
+  if (!input.image) throw new Error("--env daytona needs --daytona-image");
+  if (!Number.isInteger(cpu) || cpu < 1 || cpu > 4) throw new Error(`--daytona-cpu must be an integer from 1 to 4, got ${input.cpu}`);
+  // The warm fixture pins its lifecycle; inherit removes the pins. The two contradict.
+  if (input.warm && input.inherit) throw new Error("--daytona-warm and --daytona-inherit cannot be combined");
+  return { env, cpu };
+}

tests/restart-chaos/ttft-baseline.ts

@@ -30,7 +30,7 @@ import { DaytonaWaveSweeper } from "./daytona-sweep.js";
 import { Timeline } from "./timeline.js";
 import type { AdapterId, FaultKind } from "./types.js";
 import {
-  baselineRows, rankPreTurnSteps, recoverySamples, distribution, openShellExecutions,
+  baselineRows, rankPreTurnSteps, recoverySamples, distribution, openShellExecutions, daytonaBaselineHolders, parseBaselineEnvironment,
   type BaselineEvent, type BaselineFault, type BaselineRun,
 } from "./ttft-baseline-stats.js";
 
@@ -269,20 +269,12 @@ async function main() {
   const requested = values.adapters!.split(",").map(value => value.trim()) as AdapterId[];
   const samples = Math.max(1, Number(values.samples));
   const restarts = Math.max(0, Number(values.restarts));
-  const env = values.env === "daytona" ? "daytona" as const : "local" as const;
-  if (env === "daytona" && !values["daytona-image"]) throw new Error("--env daytona needs --daytona-image");
+  const { env, cpu } = parseBaselineEnvironment({ env: values.env, image: values["daytona-image"], cpu: values["daytona-cpu"],
+    warm: values["daytona-warm"] === true, inherit: values["daytona-inherit"] === true });
   if (env === "daytona") {
-    // A warm sandbox (the warm fixture, or the inheriting default) outlives its task for
-    // the idle window and no other task reuses it, so count one per sampled task.
     const budget = Number(process.env.RESTART_CHAOS_DAYTONA_VCPU_BUDGET ?? DAYTONA_VCPU_BUDGET);
-    const perTaskPhases = ["coldwarm", "midturn", "verify"].filter(phase => phases.has(phase)).length
-      + (phases.has("restart") ? Math.max(1, Number(values.restarts)) : 0);
-    const cpu = Number(values["daytona-cpu"]);
-    // The default fixture releases each sandbox after its turn; only a warm one is retained.
     const retainsSandboxes = values["daytona-warm"] === true || values["daytona-inherit"] === true;
-    const holders = retainsSandboxes
-      ? requested.length * Math.max(1, Number(values.samples)) * Math.max(1, perTaskPhases)
-      : requested.length;
+    const holders = daytonaBaselineHolders({ adapters: requested.length, samples, phases, restarts, retainsSandboxes });
     if (holders * cpu > budget) {
       throw new Error(`${holders} ${retainsSandboxes ? "sampled tasks (warm sandboxes outlive their task)" : "concurrent tasks"} × --daytona-cpu ${cpu} vCPU can be held at once, `
         + `exceeding the Daytona vCPU budget of ${budget}. Lower --samples, --adapters or --daytona-cpu (RESTART_CHAOS_DAYTONA_VCPU_BUDGET sets the budget).`);
@@ -360,7 +352,7 @@ async function main() {
     const api = new ChaosApi(supervisor.baseUrl);
     const repositoryPath = values["workspace-repo"] ? path.resolve(values["workspace-repo"]) : undefined;
     projectScopedTasks = Boolean(repositoryPath);
-    const lab = await seedChaosLab({ api, env, daytonaImage: values["daytona-image"], daytonaCpu: Number(values["daytona-cpu"]),
+    const lab = await seedChaosLab({ api, env, daytonaImage: values["daytona-image"], daytonaCpu: cpu,
       daytonaWarm: values["daytona-warm"], daytonaInherit: values["daytona-inherit"],
       adapters: requested, concurrency: requested.length, budgetCents: 3000,
       masks: ["rc06"], fixtureRoot: supervisor.paths.temporaryRoot, waveId, timeline, adapterConfigOverrides, repositoryPath,
@@ -461,8 +453,10 @@ async function main() {
     // First turn after an idle control-plane restart: K1 while nothing runs,
     // then one more follow-up on each task the cold/warm phase left at rest.
     // The fault must be idle: no timed-out sample may still be running.
+    // A failed runs read counts as not at rest: the fault must never land on a live turn.
     const allAtRest = timedOutIssues.size === 0 && (await Promise.all(tasks.map(async task =>
-      (await runsFor(api, task.issueId)).every(run => !ACTIVE_RUN_STATUSES.has(String(run.status)))))).every(Boolean);
+      (await api.get<Json[]>(`/api/issues/${task.issueId}/runs`, 10_000).catch(() => null))
+        ?.every(run => !ACTIVE_RUN_STATUSES.has(String(run.status))) ?? false))).every(Boolean);
     if (phases.has("postfault") && followUpTasks.size && !allAtRest) {
       timeline.add("baseline_postfault_skipped", { reason: "work_not_at_rest", timedOut: timedOutIssues.size });
     }