diff --git a/tests/restart-chaos/seed.test.ts b/tests/restart-chaos/seed.test.ts index 20c0437b59e..de1310c11d9 100644 --- a/tests/restart-chaos/seed.test.ts +++ b/tests/restart-chaos/seed.test.ts @@ -3,7 +3,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { ChaosApi } from "./api.js"; -import { seedChaosLab } from "./seed.js"; +import { inheritingEnvironmentConfig, seedChaosLab } from "./seed.js"; import { Timeline } from "./timeline.js"; type Call = { method: string; route: string; body?: unknown }; @@ -104,3 +104,11 @@ describe("seedChaosLab RC-06 mask", () => { expect(timeline.entries.some(entry => entry.kind === "seed_mask_rc06")).toBe(false); }); }); + +describe("inheritingEnvironmentConfig", () => { + it("drops the lifecycle and lease pins and keeps everything else", () => { + const fixture = { provider: "daytona", image: "img", runnerLifecycleMode: "per_turn", reuseLease: false }; + expect(inheritingEnvironmentConfig(fixture)).toEqual({ provider: "daytona", image: "img" }); + expect(fixture).toHaveProperty("runnerLifecycleMode", "per_turn"); + }); +}); diff --git a/tests/restart-chaos/seed.ts b/tests/restart-chaos/seed.ts index adc7c345430..c7dae098048 100644 --- a/tests/restart-chaos/seed.ts +++ b/tests/restart-chaos/seed.ts @@ -59,6 +59,12 @@ export function chaosDaytonaShape(cpu: number) { return { cpu, memory: 4, disk: 10 }; } +/** A user's default environment config: the lifecycle and lease reuse are left to the agent. */ +export function inheritingEnvironmentConfig(config: Record): Record { + const { runnerLifecycleMode: _lifecycle, reuseLease: _reuse, ...rest } = config; + return rest; +} + async function createDaytonaEnvironment(input: { api: ChaosApi; companyId: string; image: string; cpu: number; reuseLease: boolean; waveId: string; secretRefs: SecretReferenceMap; timeline: Timeline; warm?: boolean; inherit?: boolean }): Promise { // The warm fixture keeps a reusable sandbox and a warm runner between turns. @@ -69,13 +75,9 @@ async function createDaytonaEnvironment(input: { api: ChaosApi; companyId: strin const payload = fixture.buildEnvironment({ secretRefs: input.secretRefs, daytonaImage: input.image, executionId: input.waveId }); const shape = chaosDaytonaShape(input.cpu); // A user's default environment: the agent decides the lifecycle, and nothing pins lease reuse. - if (input.inherit) { - const config = payload.config as Record; - delete config.runnerLifecycleMode; - delete config.reuseLease; - } + const baseConfig = input.inherit ? inheritingEnvironmentConfig(payload.config as Record) : payload.config as Record; const environment = await input.api.post<{ id: string; driver: string }>(`/api/companies/${input.companyId}/environments`, - { ...payload, config: { ...(payload.config as Record), ...shape, ...(input.reuseLease ? { reuseLease: true } : {}) } }); + { ...payload, config: { ...baseConfig, ...shape, ...(input.reuseLease ? { reuseLease: true } : {}) } }); input.timeline.add("seed_environment", { environmentId: environment.id, driver: environment.driver, image: input.image, ...shape, reuseLease: input.reuseLease }); return environment.id; diff --git a/tests/restart-chaos/ttft-baseline-stats.test.ts b/tests/restart-chaos/ttft-baseline-stats.test.ts index 2a1b7173b96..e5f04482c6d 100644 --- a/tests/restart-chaos/ttft-baseline-stats.test.ts +++ b/tests/restart-chaos/ttft-baseline-stats.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { - baselineRows, distribution, openShellExecutions, percentile, rankPreTurnSteps, recoverySamples, runMetrics, runWarmth, + baselineRows, daytonaBaselineHolders, distribution, parseBaselineEnvironment, openShellExecutions, percentile, rankPreTurnSteps, recoverySamples, runMetrics, runWarmth, type BaselineEvent, type BaselineRun, } from "./ttft-baseline-stats.js"; @@ -77,6 +77,31 @@ describe("ttft baseline statistics", () => { ttft: { n: 2, p50: 6_000, p95: 12_000 } }); }); + it("leaves post-restart and verification runs out of the baseline rows", () => { + const rows = baselineRows([coldRun("a"), run("p", "postfault", [span("task.time_to_first_token", 0, 20_000)]), + run("v", "verify", [span("task.time_to_first_token", 0, 30_000)])]); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ warmth: "cold", ttft: { n: 1, p50: 6_000 } }); + }); + + it("counts held Daytona sandboxes per concurrent task, or per sampled task when they stay warm", () => { + const phases = new Set(["coldwarm", "midturn", "restart"]); + expect(daytonaBaselineHolders({ adapters: 3, samples: 5, phases, restarts: 2, retainsSandboxes: false })).toBe(3); + expect(daytonaBaselineHolders({ adapters: 3, samples: 5, phases, restarts: 2, retainsSandboxes: true })).toBe(60); + expect(daytonaBaselineHolders({ adapters: 2, samples: 0, phases: new Set(["postfault"]), restarts: 0, retainsSandboxes: true })).toBe(2); + }); + + it("rejects contradictory or out-of-range environment options", () => { + const base = { env: "daytona", image: "img", cpu: "4", warm: false, inherit: false }; + expect(parseBaselineEnvironment(base)).toEqual({ env: "daytona", cpu: 4 }); + expect(parseBaselineEnvironment({ ...base, env: "local", image: undefined })).toMatchObject({ env: "local" }); + expect(() => parseBaselineEnvironment({ ...base, env: "dayton" })).toThrow(/local or daytona/); + expect(() => parseBaselineEnvironment({ ...base, image: undefined })).toThrow(/needs --daytona-image/); + for (const cpu of ["0", "5", "abc", "2.5"]) expect(() => parseBaselineEnvironment({ ...base, cpu })).toThrow(/1 to 4/); + expect(() => parseBaselineEnvironment({ ...base, warm: true, inherit: true })).toThrow(/cannot be combined/); + expect(() => parseBaselineEnvironment({ ...base, env: "local", image: undefined, warm: true })).toThrow(/only to --env daytona/); + }); + it("ranks pre-turn steps by p95 without cumulative wrappers", () => { const ranked = rankPreTurnSteps([coldRun("a"), coldRun("b", 2)]); expect(ranked[0]).toMatchObject({ step: "runner.process.launch", n: 2, p95: 3_000 }); diff --git a/tests/restart-chaos/ttft-baseline-stats.ts b/tests/restart-chaos/ttft-baseline-stats.ts index 47caa34ae82..69b89ae867c 100644 --- a/tests/restart-chaos/ttft-baseline-stats.ts +++ b/tests/restart-chaos/ttft-baseline-stats.ts @@ -235,3 +235,31 @@ export function openShellExecutions(events: readonly BaselineEvent[], marker?: s } return [...open.entries()].map(([executionId, entry]) => ({ executionId, matched: entry.matched, unlabelled: !entry.labelled })); } + +/** Daytona sandboxes a TTFT baseline can hold at once, for the vCPU budget check. */ +export function daytonaBaselineHolders(input: { adapters: number; samples: number; phases: ReadonlySet; restarts: number; + retainsSandboxes: boolean }): number { + // The default fixture releases each sandbox after its turn, so only the concurrent tasks hold one. + if (!input.retainsSandboxes) return input.adapters; + // A warm sandbox outlives its task for the idle window and no other task reuses it. + const perTaskPhases = ["coldwarm", "midturn", "verify"].filter(phase => input.phases.has(phase)).length + + (input.phases.has("restart") ? Math.max(1, input.restarts) : 0); + return input.adapters * Math.max(1, input.samples) * Math.max(1, perTaskPhases); +} + +/** Validates the TTFT driver's environment options; returns the environment. */ +export function parseBaselineEnvironment(input: { env: string | undefined; image: string | undefined; cpu: string | undefined; + warm: boolean; inherit: boolean }): { env: "local" | "daytona"; cpu: number } { + const env = input.env ?? "local"; + if (env !== "local" && env !== "daytona") throw new Error(`--env must be local or daytona, got ${env}`); + const cpu = Number(input.cpu ?? "4"); + if (env === "local") { + if (input.warm || input.inherit || input.image) throw new Error("--daytona-* options apply only to --env daytona"); + return { env, cpu }; + } + if (!input.image) throw new Error("--env daytona needs --daytona-image"); + if (!Number.isInteger(cpu) || cpu < 1 || cpu > 4) throw new Error(`--daytona-cpu must be an integer from 1 to 4, got ${input.cpu}`); + // The warm fixture pins its lifecycle; inherit removes the pins. The two contradict. + if (input.warm && input.inherit) throw new Error("--daytona-warm and --daytona-inherit cannot be combined"); + return { env, cpu }; +} diff --git a/tests/restart-chaos/ttft-baseline.ts b/tests/restart-chaos/ttft-baseline.ts index 28e29aa9535..35422c604a6 100644 --- a/tests/restart-chaos/ttft-baseline.ts +++ b/tests/restart-chaos/ttft-baseline.ts @@ -30,7 +30,7 @@ import { DaytonaWaveSweeper } from "./daytona-sweep.js"; import { Timeline } from "./timeline.js"; import type { AdapterId, FaultKind } from "./types.js"; import { - baselineRows, rankPreTurnSteps, recoverySamples, distribution, openShellExecutions, + baselineRows, rankPreTurnSteps, recoverySamples, distribution, openShellExecutions, daytonaBaselineHolders, parseBaselineEnvironment, type BaselineEvent, type BaselineFault, type BaselineRun, } from "./ttft-baseline-stats.js"; @@ -269,20 +269,12 @@ async function main() { const requested = values.adapters!.split(",").map(value => value.trim()) as AdapterId[]; const samples = Math.max(1, Number(values.samples)); const restarts = Math.max(0, Number(values.restarts)); - const env = values.env === "daytona" ? "daytona" as const : "local" as const; - if (env === "daytona" && !values["daytona-image"]) throw new Error("--env daytona needs --daytona-image"); + const { env, cpu } = parseBaselineEnvironment({ env: values.env, image: values["daytona-image"], cpu: values["daytona-cpu"], + warm: values["daytona-warm"] === true, inherit: values["daytona-inherit"] === true }); if (env === "daytona") { - // A warm sandbox (the warm fixture, or the inheriting default) outlives its task for - // the idle window and no other task reuses it, so count one per sampled task. const budget = Number(process.env.RESTART_CHAOS_DAYTONA_VCPU_BUDGET ?? DAYTONA_VCPU_BUDGET); - const perTaskPhases = ["coldwarm", "midturn", "verify"].filter(phase => phases.has(phase)).length - + (phases.has("restart") ? Math.max(1, Number(values.restarts)) : 0); - const cpu = Number(values["daytona-cpu"]); - // The default fixture releases each sandbox after its turn; only a warm one is retained. const retainsSandboxes = values["daytona-warm"] === true || values["daytona-inherit"] === true; - const holders = retainsSandboxes - ? requested.length * Math.max(1, Number(values.samples)) * Math.max(1, perTaskPhases) - : requested.length; + const holders = daytonaBaselineHolders({ adapters: requested.length, samples, phases, restarts, retainsSandboxes }); if (holders * cpu > budget) { throw new Error(`${holders} ${retainsSandboxes ? "sampled tasks (warm sandboxes outlive their task)" : "concurrent tasks"} × --daytona-cpu ${cpu} vCPU can be held at once, ` + `exceeding the Daytona vCPU budget of ${budget}. Lower --samples, --adapters or --daytona-cpu (RESTART_CHAOS_DAYTONA_VCPU_BUDGET sets the budget).`); @@ -360,7 +352,7 @@ async function main() { const api = new ChaosApi(supervisor.baseUrl); const repositoryPath = values["workspace-repo"] ? path.resolve(values["workspace-repo"]) : undefined; projectScopedTasks = Boolean(repositoryPath); - const lab = await seedChaosLab({ api, env, daytonaImage: values["daytona-image"], daytonaCpu: Number(values["daytona-cpu"]), + const lab = await seedChaosLab({ api, env, daytonaImage: values["daytona-image"], daytonaCpu: cpu, daytonaWarm: values["daytona-warm"], daytonaInherit: values["daytona-inherit"], adapters: requested, concurrency: requested.length, budgetCents: 3000, masks: ["rc06"], fixtureRoot: supervisor.paths.temporaryRoot, waveId, timeline, adapterConfigOverrides, repositoryPath, @@ -461,8 +453,10 @@ async function main() { // First turn after an idle control-plane restart: K1 while nothing runs, // then one more follow-up on each task the cold/warm phase left at rest. // The fault must be idle: no timed-out sample may still be running. + // A failed runs read counts as not at rest: the fault must never land on a live turn. const allAtRest = timedOutIssues.size === 0 && (await Promise.all(tasks.map(async task => - (await runsFor(api, task.issueId)).every(run => !ACTIVE_RUN_STATUSES.has(String(run.status)))))).every(Boolean); + (await api.get(`/api/issues/${task.issueId}/runs`, 10_000).catch(() => null)) + ?.every(run => !ACTIVE_RUN_STATUSES.has(String(run.status))) ?? false))).every(Boolean); if (phases.has("postfault") && followUpTasks.size && !allAtRest) { timeline.add("baseline_postfault_skipped", { reason: "work_not_at_rest", timedOut: timedOutIssues.size }); }