From 5d39e40d6f5cdd194233cbe2ea43e7fa68c6a0b0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 8 Nov 2026 08:15:30 -0600 Subject: [PATCH 1/3] test(restart-chaos): measure TTFT in a real repository workspace, after an idle restart, and on Daytona - --workspace-repo seeds a real repository as the project's primary workspace; tasks then run in its isolated git worktree. - postfault phase: the first follow-up after an idle K1, reported on its own. - --env daytona with --daytona-image/--daytona-cpu, --daytona-warm (reusable warm fixture) and --daytona-inherit (a user's default environment); every sandbox is released before the server stops, on signals too. - Per-run workspace operation durations; chaos main.ts takes the same Daytona fixture options. Co-Authored-By: Claude Opus 5.5 --- tests/restart-chaos/config.ts | 16 ++- tests/restart-chaos/daytona-controller.ts | 51 ++++++++ tests/restart-chaos/main.ts | 1 + tests/restart-chaos/seed.ts | 24 +++- tests/restart-chaos/ttft-baseline-stats.ts | 6 +- tests/restart-chaos/ttft-baseline.ts | 136 +++++++++++++++++++-- tests/restart-chaos/types.ts | 4 + 7 files changed, 218 insertions(+), 20 deletions(-) create mode 100644 tests/restart-chaos/daytona-controller.ts diff --git a/tests/restart-chaos/config.ts b/tests/restart-chaos/config.ts index ff4196ba3c8..269240abce2 100644 --- a/tests/restart-chaos/config.ts +++ b/tests/restart-chaos/config.ts @@ -84,6 +84,9 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC "daytona-image": { type: "string" }, "daytona-cpu": { type: "string" }, "daytona-reuse-lease": { type: "boolean" }, + // The reusable warm Daytona fixture, or an environment that inherits the lifecycle (a user's default). + "daytona-warm": { type: "boolean" }, + "daytona-inherit": { type: "boolean" }, keep: { type: "boolean" }, "rc06-masked": { type: "boolean" }, "storm-minutes": { type: "string" }, @@ -96,7 +99,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC const pollMs = integer(values["poll-ms"], "poll-ms", 3_000, 2_000, 5_000); const budgetCents = integer(values["budget-cents"], "budget-cents", 3_000, 1, 100_000); if (values.replay !== undefined) { - for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "daytona-reuse-lease", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point"] as const) { + for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "daytona-reuse-lease", "daytona-warm", "daytona-inherit", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point"] as const) { if (values[key] !== undefined) throw new Error(`--replay reproduces the saved plan; do not combine it with --${key}`); } return { mode: "replay", ref: parseReplayRef(values.replay), keep, @@ -142,7 +145,8 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC budgetCents, pollMs, ...(env === "daytona" ? { daytonaImage, daytonaCpu: integer(values["daytona-cpu"], "daytona-cpu", DAYTONA_DEFAULT_CPU, 1, 4), - ...(values["daytona-reuse-lease"] ? { daytonaReuseLease: true } : {}) } : {}), + ...(values["daytona-reuse-lease"] ? { daytonaReuseLease: true } : {}), + ...(values["daytona-warm"] ? { daytonaWarm: true } : {}), ...(values["daytona-inherit"] ? { daytonaInherit: true } : {}) } : {}), ...(values["rc06-masked"] ? { masks: ["rc06"] } : {}), ...(lifecycle ? { lifecycle } : {}), // Seeded K4 hold upper end; the controller-outage cap is 600 s (#19797). @@ -154,7 +158,13 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC if (env !== "daytona" && values["daytona-reuse-lease"]) throw new Error("--daytona-reuse-lease applies only to --env daytona"); if (options.daytonaCpu !== undefined) { const budget = integer(process.env.RESTART_CHAOS_DAYTONA_VCPU_BUDGET, "RESTART_CHAOS_DAYTONA_VCPU_BUDGET", DAYTONA_VCPU_BUDGET, 1, 1_000); - if (options.concurrency * options.daytonaCpu > budget) { + // A warm sandbox outlives its task for the idle window, and a new task cannot reuse + // another task's sandbox, so with --daytona-warm every task can hold one at once. + const holders = options.daytonaWarm ? Math.max(options.concurrency, options.tasks) : options.concurrency; + if (holders * options.daytonaCpu > budget) { + if (options.daytonaWarm) throw new Error(`--daytona-warm keeps each task's sandbox for the idle window: ${holders} tasks × ` + + `--daytona-cpu ${options.daytonaCpu} vCPU exceeds the Daytona vCPU budget of ${budget}. Lower --tasks or --daytona-cpu ` + + "(RESTART_CHAOS_DAYTONA_VCPU_BUDGET sets the budget)."); throw new Error(`--concurrency ${options.concurrency} × --daytona-cpu ${options.daytonaCpu} vCPU exceeds the Daytona vCPU budget of ${budget} ` + "for chaos tasks. Lower one of them (RESTART_CHAOS_DAYTONA_VCPU_BUDGET sets the budget)."); } diff --git a/tests/restart-chaos/daytona-controller.ts b/tests/restart-chaos/daytona-controller.ts new file mode 100644 index 00000000000..42b519c2c91 --- /dev/null +++ b/tests/restart-chaos/daytona-controller.ts @@ -0,0 +1,51 @@ +import { spawnSync } from "node:child_process"; +import { writeFileSync } from "node:fs"; +import path from "node:path"; +import { assertControllerMatchesImage, defaultDaytonaImageCacheRoot, prepareDaytonaImageArtifacts } from "./daytona-image.js"; +import { PLUGIN_BUILD_STAMP, pluginBuildState } from "./plugin-build.js"; +import type { Timeline } from "./timeline.js"; + +function run(command: string, args: string[], cwd: string, label: string) { + const result = spawnSync(command, args, { cwd, stdio: "inherit" }); + if (result.status !== 0) throw new Error(`${label} failed (${command} ${args.join(" ")})`); +} + +/** + * Mirrors restart-chaos main.ts's Daytona preparation for the TTFT driver. + * The bundled Daytona plugin is outside the root workspace; prepare it as CI + * does, and rebuild it whenever its sources (or the plugin SDK's) changed since + * the last build, so a worktree that moved to a new master tests that master's + * plugin rather than an older build. + */ +export function ensureDaytonaPlugin(repositoryRoot: string, timeline: Timeline) { + const root = path.join(repositoryRoot, "packages/plugins/sandbox-providers/daytona"); + // Inputs outside the plugin directory: the linked plugin SDK and the tsconfig chain both packages extend. + const extraInputs = [ + ...["src", "package.json", "tsconfig.json"].map(file => path.join(repositoryRoot, "packages/plugins/sdk", file)), + ...["tsconfig.json", "tsconfig.base.json"].map(file => path.join(repositoryRoot, file)), + ]; + const state = pluginBuildState(root, extraInputs); + timeline.add("daytona_plugin_build_state", { reason: state.reason, inputHash: state.inputHash }); + if (!state.needsBuild) return; + run("pnpm", ["install", "--ignore-workspace", "--frozen-lockfile", "--ignore-scripts"], root, "Daytona plugin install"); + run(process.execPath, [path.join(repositoryRoot, "scripts/link-plugin-dev-sdk.mjs")], repositoryRoot, "Plugin SDK link"); + run("pnpm", ["--dir", root, "build"], repositoryRoot, "Daytona plugin build"); + writeFileSync(path.join(root, PLUGIN_BUILD_STAMP), `${state.inputHash}\n`); + timeline.add("daytona_plugin_built", { inputHash: state.inputHash }); +} + +/** + * Controller environment for `--env daytona`: extract the image's Linux runnerd + * and provider pack (cached per image digest, no Docker), refuse a controller + * whose runner source differs from the image, and build the Daytona plugin. + */ +export async function prepareDaytonaController(input: { image: string; repositoryRoot: string; timeline: Timeline }): + Promise> { + const artifacts = await prepareDaytonaImageArtifacts({ image: input.image, cacheRoot: defaultDaytonaImageCacheRoot(), + log: line => console.info(`[daytona-image] ${line}`) }); + const { localContentId } = await assertControllerMatchesImage(artifacts, input.repositoryRoot); + input.timeline.add("daytona_image_matched", { image: input.image, contentId: artifacts.contentId, localContentId, sourceRevision: artifacts.sourceRevision, + runnerdSha256: artifacts.runnerdSha256, providerPackDigest: artifacts.providerPackDigest, platformManifestDigest: artifacts.platformManifestDigest }); + ensureDaytonaPlugin(input.repositoryRoot, input.timeline); + return { PAPERCLIP_RUNNER_REMOTE_BINARY_PATH: artifacts.runnerdPath, PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH: artifacts.providerPackPath }; +} diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index 30ca3c0a045..faa71aede87 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -207,6 +207,7 @@ async function main(): Promise { const api = new ChaosApi(supervisor.baseUrl); const lab = await seedChaosLab({ api, env: plan.options.env, daytonaImage: plan.options.daytonaImage, daytonaCpu: plan.options.daytonaCpu, daytonaReuseLease: plan.options.daytonaReuseLease, + daytonaWarm: plan.options.daytonaWarm, daytonaInherit: plan.options.daytonaInherit, adapters: [...new Set(plan.tasks.map(task => task.adapter))], concurrency: plan.options.concurrency, budgetCents: plan.options.budgetCents, masks: plan.options.masks, fixtureRoot: path.join(supervisor.paths.temporaryRoot, "fixtures"), waveId: plan.waveId, timeline, diff --git a/tests/restart-chaos/seed.ts b/tests/restart-chaos/seed.ts index 8c1f53b1ae8..adc7c345430 100644 --- a/tests/restart-chaos/seed.ts +++ b/tests/restart-chaos/seed.ts @@ -2,7 +2,7 @@ import { execFileSync } from "node:child_process"; import { chmod, mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import type { RunnerApi } from "../runner-e2e/api.js"; -import { runnerEnvironments, runnerProfiles } from "../runner-e2e/catalog.js"; +import { daytonaWarmEnvironment, runnerEnvironments, runnerProfiles } from "../runner-e2e/catalog.js"; import { installFixtureDaytonaProvider, waitForFixtureDaytonaProvider } from "../runner-e2e/live-fixtures.js"; import type { MatrixExecution, SecretReference, SecretReferenceMap } from "../runner-e2e/types.js"; import type { ChaosApi } from "./api.js"; @@ -60,13 +60,20 @@ export function chaosDaytonaShape(cpu: number) { } async function createDaytonaEnvironment(input: { api: ChaosApi; companyId: string; image: string; cpu: number; reuseLease: boolean; waveId: string; - secretRefs: SecretReferenceMap; timeline: Timeline }): Promise { - const fixture = runnerEnvironments.find(candidate => candidate.id === "daytona"); + secretRefs: SecretReferenceMap; timeline: Timeline; warm?: boolean; inherit?: boolean }): Promise { + // The warm fixture keeps a reusable sandbox and a warm runner between turns. + const fixture = input.warm ? daytonaWarmEnvironment : runnerEnvironments.find(candidate => candidate.id === "daytona"); if (!fixture) throw new Error("runner-e2e catalog has no daytona environment fixture"); const view = runnerApiView(input.api); await waitForFixtureDaytonaProvider(view, { companyId: input.companyId, deadlineAt: Date.now() + 120_000 }); const payload = fixture.buildEnvironment({ secretRefs: input.secretRefs, daytonaImage: input.image, executionId: input.waveId }); const shape = chaosDaytonaShape(input.cpu); + // A user's default environment: the agent decides the lifecycle, and nothing pins lease reuse. + if (input.inherit) { + const config = payload.config as Record; + delete config.runnerLifecycleMode; + delete config.reuseLease; + } const environment = await input.api.post<{ id: string; driver: string }>(`/api/companies/${input.companyId}/environments`, { ...payload, config: { ...(payload.config as Record), ...shape, ...(input.reuseLease ? { reuseLease: true } : {}) } }); input.timeline.add("seed_environment", { environmentId: environment.id, driver: environment.driver, image: input.image, ...shape, @@ -120,6 +127,12 @@ export async function seedChaosLab(input: { timeline: Timeline; /** Merged into each agent's adapterConfig (for example a runner lifecycle mode). */ adapterConfigOverrides?: Record; + /** Use the reusable warm Daytona environment fixture (reuseLease, warm runner). */ + daytonaWarm?: boolean; + /** Use a Daytona environment that inherits the lifecycle and leaves lease reuse unset, as a user's default does. */ + daytonaInherit?: boolean; + /** A real repository to use as the project's primary workspace instead of the tiny fixture. */ + repositoryPath?: string; /** Called as soon as the wave owns external resources that need cleanup. */ onEnvironment?: (lab: Pick) => void; }): Promise { @@ -130,7 +143,8 @@ export async function seedChaosLab(input: { await installFixtureDaytonaProvider(runnerApiView(input.api), { environment: { id: "daytona" } } as MatrixExecution); timeline.add("seed_daytona_plugin", {}); } - const fixtureRepositoryPath = await createFixtureRepository(path.join(input.fixtureRoot, "chaos-sandbox")); + const fixtureRepositoryPath = input.repositoryPath + ?? await createFixtureRepository(path.join(input.fixtureRoot, "chaos-sandbox")); const company = await api.post<{ id: string }>("/api/companies", { name: "Chaos Lab", description: `Restart chaos harness wave ${input.waveId}`, @@ -161,7 +175,7 @@ export async function seedChaosLab(input: { let environmentId: string; if (input.env === "daytona") { environmentId = await createDaytonaEnvironment({ api, companyId: company.id, image: input.daytonaImage!, cpu: input.daytonaCpu ?? 4, - reuseLease: input.daytonaReuseLease === true, waveId: input.waveId, secretRefs, timeline }); + reuseLease: input.daytonaReuseLease === true, waveId: input.waveId, secretRefs, timeline, warm: input.daytonaWarm, inherit: input.daytonaInherit }); input.onEnvironment?.({ companyId: company.id, environmentId, environmentKind: "daytona" }); if (input.adapters.includes("codex")) await qualifyNativeCodex({ api, companyId: company.id, environmentId, timeline }); } else { diff --git a/tests/restart-chaos/ttft-baseline-stats.ts b/tests/restart-chaos/ttft-baseline-stats.ts index 9f90c0ecad6..47caa34ae82 100644 --- a/tests/restart-chaos/ttft-baseline-stats.ts +++ b/tests/restart-chaos/ttft-baseline-stats.ts @@ -16,7 +16,7 @@ export interface BaselineRun { adapter: string; environment: string; /** Driver intent: a new task, a follow-up comment on the same task, or a run crossed by a restart. */ - phase: "cold" | "warm" | "restart" | "midturn" | "verify"; + phase: "cold" | "warm" | "restart" | "midturn" | "verify" | "postfault"; events: BaselineEvent[]; } @@ -149,8 +149,8 @@ export interface BaselineRow { adapter: string; environment: string; warmth: "co export function baselineRows(runs: readonly BaselineRun[]): BaselineRow[] { const groups = new Map(); for (const run of runs) { - // Restart samples and new-agent verification samples are reported on their own. - if (run.phase === "restart" || run.phase === "verify") continue; + // Restart, post-restart and new-agent verification samples are reported on their own. + if (run.phase === "restart" || run.phase === "verify" || run.phase === "postfault") continue; const warmth = runWarmth(run); const key = `${run.adapter}|${run.environment}|${warmth}`; const group = groups.get(key) ?? { adapter: run.adapter, environment: run.environment, warmth, metrics: [] }; diff --git a/tests/restart-chaos/ttft-baseline.ts b/tests/restart-chaos/ttft-baseline.ts index 291a12ffd19..28e29aa9535 100644 --- a/tests/restart-chaos/ttft-baseline.ts +++ b/tests/restart-chaos/ttft-baseline.ts @@ -20,11 +20,13 @@ import path from "node:path"; import { parseArgs } from "node:util"; import { setTimeout as delay } from "node:timers/promises"; import { ChaosApi } from "./api.js"; -import { CREDENTIAL_FOR_ADAPTER } from "./config.js"; +import { CREDENTIAL_FOR_ADAPTER, DAYTONA_VCPU_BUDGET } from "./config.js"; import { ACTIVE_RUN_STATUSES } from "./grader.js"; import { chaosLifecycleOverrides, seedChaosLab, type ChaosLab } from "./seed.js"; import { ChaosSupervisor } from "./supervisor.js"; import { createDb } from "../../packages/db/src/index.js"; +import { prepareDaytonaController } from "./daytona-controller.js"; +import { DaytonaWaveSweeper } from "./daytona-sweep.js"; import { Timeline } from "./timeline.js"; import type { AdapterId, FaultKind } from "./types.js"; import { @@ -88,9 +90,13 @@ async function steerQueuedComment(api: ChaosApi, issueId: string, commentId: str return { acknowledgedAt: null, disposition }; } +/** Set by --workspace-repo: tasks then run in the project's isolated workspace of that repository. */ +let projectScopedTasks = false; + async function createTask(api: ChaosApi, lab: ChaosLab, adapter: AdapterId, title: string, description: string) { const issue = await api.post(`/api/companies/${lab.companyId}/issues`, { title, description, status: "todo", priority: "medium", assigneeAgentId: lab.agents[adapter].id, + ...(projectScopedTasks ? { projectId: lab.projectId } : {}), }); return String(issue.id); } @@ -200,6 +206,11 @@ function report(runs: BaselineRun[], faults: BaselineFault[]): string { lines.push("", "New agent behind harness verification (TTFT counts from run creation; the verification wait precedes it):", "", lines[0]!, lines[1]!); for (const row of baselineRows(verifyRuns)) lines.push(rowLine(row)); } + const postfault = runs.filter(run => run.phase === "postfault").map(run => ({ ...run, phase: "cold" as const })); + if (postfault.length) { + lines.push("", "First follow-up after an idle control-plane restart (K1):", "", lines[0]!, lines[1]!); + for (const row of baselineRows(postfault)) lines.push(rowLine(row)); + } lines.push("", "| Pre-turn step | n | p50 | p95 | Adapters |", "|---|---|---|---|---|"); for (const step of rankPreTurnSteps(runs).slice(0, 25)) lines.push(`| \`${step.step}\` | ${step.n} | ${ms(step.p50)} | ${ms(step.p95)} | ${step.adapters.join(", ")} |`); @@ -236,6 +247,17 @@ async function main() { "midturn-fault": { type: "string" }, // verify phase: fault the control plane while a newly hired agent runs harness verification. "verify-fault": { type: "string" }, + // A real repository (for example a fresh clone of paperclip) as the project's + // primary workspace; every task then runs in its isolated git worktree. + "workspace-repo": { type: "string" }, + // local (default) or daytona; daytona needs --daytona-image (the signed Runner image). + env: { type: "string", default: "local" }, + "daytona-image": { type: "string" }, + "daytona-cpu": { type: "string", default: "4" }, + // Use the reusable warm Daytona environment (reuseLease, warm runner) instead of the per-turn one. + "daytona-warm": { type: "boolean", default: false }, + // A Daytona environment as a user creates it by default: lifecycle inherited, lease reuse unset. + "daytona-inherit": { type: "boolean", default: false }, "idle-ms": { type: "string", default: "600000" }, } }); if (values.report) { @@ -247,7 +269,27 @@ async function main() { const requested = values.adapters!.split(",").map(value => value.trim()) as AdapterId[]; const samples = Math.max(1, Number(values.samples)); const restarts = Math.max(0, Number(values.restarts)); - const missing = requested.map(adapter => CREDENTIAL_FOR_ADAPTER[adapter]).filter(key => !process.env[key]); + const env = values.env === "daytona" ? "daytona" as const : "local" as const; + if (env === "daytona" && !values["daytona-image"]) throw new Error("--env daytona needs --daytona-image"); + if (env === "daytona") { + // A warm sandbox (the warm fixture, or the inheriting default) outlives its task for + // the idle window and no other task reuses it, so count one per sampled task. + const budget = Number(process.env.RESTART_CHAOS_DAYTONA_VCPU_BUDGET ?? DAYTONA_VCPU_BUDGET); + const perTaskPhases = ["coldwarm", "midturn", "verify"].filter(phase => phases.has(phase)).length + + (phases.has("restart") ? Math.max(1, Number(values.restarts)) : 0); + const cpu = Number(values["daytona-cpu"]); + // The default fixture releases each sandbox after its turn; only a warm one is retained. + const retainsSandboxes = values["daytona-warm"] === true || values["daytona-inherit"] === true; + const holders = retainsSandboxes + ? requested.length * Math.max(1, Number(values.samples)) * Math.max(1, perTaskPhases) + : requested.length; + if (holders * cpu > budget) { + throw new Error(`${holders} ${retainsSandboxes ? "sampled tasks (warm sandboxes outlive their task)" : "concurrent tasks"} × --daytona-cpu ${cpu} vCPU can be held at once, ` + + `exceeding the Daytona vCPU budget of ${budget}. Lower --samples, --adapters or --daytona-cpu (RESTART_CHAOS_DAYTONA_VCPU_BUDGET sets the budget).`); + } + } + const missing = [...requested.map(adapter => CREDENTIAL_FOR_ADAPTER[adapter]), ...(env === "daytona" ? ["DAYTONA_API_KEY"] : [])] + .filter(key => !process.env[key]); if (missing.length) throw new Error(`Missing ${missing.join(", ")}; run: set -a; source ~/.secrets; set +a`); const midturnMode = values["midturn-mode"]; if (midturnMode !== "queue" && midturnMode !== "steer") throw new Error("--midturn-mode must be queue or steer"); @@ -276,10 +318,34 @@ async function main() { const agentLifecycles: Record = {}; const midturn: Json[] = []; const timedOutIssues = new Set(); + let sweeper: DaytonaWaveSweeper | null = null; + const workspaceOperations: Array<{ runId: string; adapter: string; phase: string; operation: string; status: string; + durationMs: number | null; command: string | null }> = []; + const followUpTasks = new Map(); // One shutdown path for normal completion and for SIGINT/SIGTERM: the server // runs in a detached process group and the embedded Postgres has no exit hook. let stopping: Promise | null = null; - const shutdown = () => (stopping ??= supervisor.stop().catch(error => { + // Daytona resources are released while the server still runs, before it stops; + // the same path serves normal completion and SIGINT/SIGTERM. + const releaseDaytona = async () => { + if (!sweeper) return; + const owned = sweeper as DaytonaWaveSweeper; + sweeper = null; + try { + const cleanup = await owned.cleanUp(); + timeline.add("daytona_cleanup", cleanup); + await writeFile(path.join(outDir, "sandbox-cleanup.json"), `${JSON.stringify(cleanup, null, 2)}\n`, { mode: 0o600 }); + if (cleanup.remainingSandboxes.length || cleanup.errors.length || cleanup.environmentDeleteBlocked || !cleanup.environmentDeleted) { + process.stderr.write(`Daytona cleanup incomplete: remaining ${cleanup.remainingSandboxes.join(", ") || "none"}; ` + + `environment deleted ${cleanup.environmentDeleted}${cleanup.environmentDeleteBlocked ? ` (blocked: ${String(cleanup.environmentDeleteBlocked)})` : ""}; ${cleanup.errors.join("; ")}\n`); + process.exitCode = 1; + } + } catch (error) { + timeline.add("daytona_cleanup_failed", { error: String(error) }); + process.exitCode = 1; + } + }; + const shutdown = () => (stopping ??= releaseDaytona().then(() => supervisor.stop()).catch(error => { process.stderr.write(`supervisor stop: ${String(error)}\n`); })); const onSignal = (signal: NodeJS.Signals) => { @@ -289,10 +355,18 @@ async function main() { process.once("SIGINT", onSignal); process.once("SIGTERM", onSignal); try { - await supervisor.start(); + await supervisor.start(env === "daytona" + ? await prepareDaytonaController({ image: values["daytona-image"]!, repositoryRoot, timeline }) : {}); const api = new ChaosApi(supervisor.baseUrl); - const lab = await seedChaosLab({ api, env: "local", adapters: requested, concurrency: requested.length, budgetCents: 3000, - masks: ["rc06"], fixtureRoot: supervisor.paths.temporaryRoot, waveId, timeline, adapterConfigOverrides }); + const repositoryPath = values["workspace-repo"] ? path.resolve(values["workspace-repo"]) : undefined; + projectScopedTasks = Boolean(repositoryPath); + const lab = await seedChaosLab({ api, env, daytonaImage: values["daytona-image"], daytonaCpu: Number(values["daytona-cpu"]), + daytonaWarm: values["daytona-warm"], daytonaInherit: values["daytona-inherit"], + adapters: requested, concurrency: requested.length, budgetCents: 3000, + masks: ["rc06"], fixtureRoot: supervisor.paths.temporaryRoot, waveId, timeline, adapterConfigOverrides, repositoryPath, + onEnvironment: owned => { + sweeper = new DaytonaWaveSweeper({ apiKey: process.env.DAYTONA_API_KEY!, companyId: owned.companyId, environmentId: owned.environmentId, api }); + } }); const adapters = await readyAdapters(api, lab, requested, timeline); if (!adapters.length) throw new Error("No agent passed lifecycle verification"); // Record what each agent actually stores: "unset" means the server resolves @@ -328,6 +402,7 @@ async function main() { continue; } timeline.add("baseline_warm_done", { adapter, issueId, runs: warm.runs.length - cold.runs.length }); + followUpTasks.set(adapter, { issueId, nonce, runs: warm.runs.length }); } })); @@ -383,6 +458,29 @@ async function main() { } } + // First turn after an idle control-plane restart: K1 while nothing runs, + // then one more follow-up on each task the cold/warm phase left at rest. + // The fault must be idle: no timed-out sample may still be running. + const allAtRest = timedOutIssues.size === 0 && (await Promise.all(tasks.map(async task => + (await runsFor(api, task.issueId)).every(run => !ACTIVE_RUN_STATUSES.has(String(run.status)))))).every(Boolean); + if (phases.has("postfault") && followUpTasks.size && !allAtRest) { + timeline.add("baseline_postfault_skipped", { reason: "work_not_at_rest", timedOut: timedOutIssues.size }); + } + if (phases.has("postfault") && followUpTasks.size && allAtRest) { + const receipt = await supervisor.fault("K1", 0, "ttft-baseline-postfault"); + faults.push({ kind: "K1", readyAt: receipt.readyAt ?? Date.now(), runIds: [] }); + await Promise.all([...followUpTasks.entries()].map(async ([adapter, entry]) => { + await api.post(`/api/issues/${entry.issueId}/comments`, { body: warmFollowUp(`${entry.nonce}-postfault`), reopen: true }); + const after = await waitForRest(api, supervisor, entry.issueId, entry.runs + 1); + for (const run of after.runs) if (!runPhase.has(runIdOf(run))) runPhase.set(runIdOf(run), "postfault"); + if (after.timedOut) { + timedOutIssues.add(entry.issueId); + timeline.add("baseline_sample_timeout", { adapter, issueId: entry.issueId, phase: "postfault" }); + } + else timeline.add("baseline_postfault_done", { adapter, issueId: entry.issueId }); + })); + } + // Mid-turn follow-up: a comment posted while the agent is inside a long // tool call. Records when the comment landed and every run of the task, // so the wait splits into "rest of the turn" and "hand-off". @@ -472,15 +570,34 @@ async function main() { if (timedOutIssues.has(task.issueId)) continue; for (const run of await runsFor(api, task.issueId)) { const runId = runIdOf(run); - runs.push({ runId, adapter: task.adapter, environment: "local", + runs.push({ runId, adapter: task.adapter, environment: env, phase: runPhase.get(runId) ?? task.phase, events: await allEvents(api, runId) }); + // Workspace commands (worktree, provision, sync) the run waited on, with their durations. + const operations = await api.get(`/api/heartbeat-runs/${runId}/workspace-operations`, 10_000).catch((error: unknown) => { + // A failed read is recorded as such, never as "no workspace operations". + workspaceOperations.push({ runId, adapter: task.adapter, phase: runPhase.get(runId) ?? task.phase, + operation: "read_failed", status: error instanceof Error ? error.message.slice(0, 160) : String(error), durationMs: null, command: null }); + return [] as Json[]; + }); + for (const operation of operations) workspaceOperations.push({ runId, adapter: task.adapter, + phase: runPhase.get(runId) ?? task.phase, operation: String(operation.phase), status: String(operation.status), + durationMs: operation.startedAt && operation.finishedAt + ? Date.parse(String(operation.finishedAt)) - Date.parse(String(operation.startedAt)) : null, + command: typeof operation.command === "string" ? operation.command.slice(0, 120) : null }); } } - const data = { waveId, lifecycle: lifecycle ?? "catalog", agentLifecycles, adapters, notReady: requested.filter(adapter => !adapters.includes(adapter)), + const data = { workspaceOperations, waveId, lifecycle: lifecycle ?? "catalog", agentLifecycles, adapters, notReady: requested.filter(adapter => !adapters.includes(adapter)), timedOutTasks: timedOutIssues.size, samples, restarts, midturn, gitSha: process.env.GIT_SHA ?? null, runs, faults }; await writeFile(path.join(outDir, "baseline-data.json"), `${JSON.stringify(data)}\n`, { mode: 0o600 }); await writeFile(path.join(outDir, "baseline-report.md"), report(runs, faults), { mode: 0o600 }); process.stdout.write(report(runs, faults)); + if (workspaceOperations.length) { + const table = ["", "| Run phase | Adapter | Workspace operation | Status | Duration | Command |", "|---|---|---|---|---|---|", + ...workspaceOperations.slice().sort((a, b) => (b.durationMs ?? 0) - (a.durationMs ?? 0)) + .map(op => `| ${op.phase} | ${op.adapter} | ${op.operation} | ${op.status} | ${op.durationMs === null ? "—" : `${(op.durationMs / 1000).toFixed(2)} s`} | ${op.command ? `\`${op.command}\`` : "—"} |`)].join("\n"); + process.stdout.write(`${table}\n`); + await writeFile(path.join(outDir, "workspace-operations.md"), `${table}\n`, { mode: 0o600 }); + } if (timedOutIssues.size) process.stdout.write(`\n${timedOutIssues.size} sample(s) timed out and were left out.\n`); // Every answered mid-turn comment must be answered exactly once. const wrongReplies = midturn.filter(sample => sample.replyCount !== 1); @@ -491,7 +608,8 @@ async function main() { if (wrongReplies.length) process.exitCode = 1; process.stdout.write(`\nWrote ${outDir}\n`); } finally { - // Always stop the wrapper-owned embedded Postgres (SysV shared memory is scarce). + // Every Daytona sandbox this run created is archived or destroyed before the server + // stops, then the wrapper-owned embedded Postgres stops (SysV shared memory is scarce). await shutdown(); process.off("SIGINT", onSignal); process.off("SIGTERM", onSignal); diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index 826882763a3..58b5b514723 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -36,6 +36,10 @@ export interface WaveOptions { daytonaCpu?: number; /** `--daytona-reuse-lease`: the chaos environment keeps its sandboxes between turns (a per-turn run stops and keeps its sandbox). */ daytonaReuseLease?: boolean; + /** `--daytona-warm`: the reusable warm Daytona environment fixture. */ + daytonaWarm?: boolean; + /** `--daytona-inherit`: a Daytona environment that inherits the lifecycle and leaves lease reuse unset. */ + daytonaInherit?: boolean; /** `--k4-hold-max-seconds`: upper end of the seeded K4 hold (default 300 s, cap 600 s). */ k4HoldMaxMs?: number; /** From 3ff56d6446bc8b4e06b02a01239edd6f52e4952d Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 8 Nov 2026 08:19:50 -0600 Subject: [PATCH 2/3] test(restart-chaos): validate TTFT driver environment options; never fault on a failed runs read Co-Authored-By: Claude Opus 5.5 --- tests/restart-chaos/seed.test.ts | 10 ++++++- tests/restart-chaos/seed.ts | 14 ++++++---- .../restart-chaos/ttft-baseline-stats.test.ts | 27 +++++++++++++++++- tests/restart-chaos/ttft-baseline-stats.ts | 28 +++++++++++++++++++ tests/restart-chaos/ttft-baseline.ts | 22 ++++++--------- 5 files changed, 79 insertions(+), 22 deletions(-) diff --git a/tests/restart-chaos/seed.test.ts b/tests/restart-chaos/seed.test.ts index 20c0437b59e..de1310c11d9 100644 --- a/tests/restart-chaos/seed.test.ts +++ b/tests/restart-chaos/seed.test.ts @@ -3,7 +3,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { ChaosApi } from "./api.js"; -import { seedChaosLab } from "./seed.js"; +import { inheritingEnvironmentConfig, seedChaosLab } from "./seed.js"; import { Timeline } from "./timeline.js"; type Call = { method: string; route: string; body?: unknown }; @@ -104,3 +104,11 @@ describe("seedChaosLab RC-06 mask", () => { expect(timeline.entries.some(entry => entry.kind === "seed_mask_rc06")).toBe(false); }); }); + +describe("inheritingEnvironmentConfig", () => { + it("drops the lifecycle and lease pins and keeps everything else", () => { + const fixture = { provider: "daytona", image: "img", runnerLifecycleMode: "per_turn", reuseLease: false }; + expect(inheritingEnvironmentConfig(fixture)).toEqual({ provider: "daytona", image: "img" }); + expect(fixture).toHaveProperty("runnerLifecycleMode", "per_turn"); + }); +}); diff --git a/tests/restart-chaos/seed.ts b/tests/restart-chaos/seed.ts index adc7c345430..c7dae098048 100644 --- a/tests/restart-chaos/seed.ts +++ b/tests/restart-chaos/seed.ts @@ -59,6 +59,12 @@ export function chaosDaytonaShape(cpu: number) { return { cpu, memory: 4, disk: 10 }; } +/** A user's default environment config: the lifecycle and lease reuse are left to the agent. */ +export function inheritingEnvironmentConfig(config: Record): Record { + const { runnerLifecycleMode: _lifecycle, reuseLease: _reuse, ...rest } = config; + return rest; +} + async function createDaytonaEnvironment(input: { api: ChaosApi; companyId: string; image: string; cpu: number; reuseLease: boolean; waveId: string; secretRefs: SecretReferenceMap; timeline: Timeline; warm?: boolean; inherit?: boolean }): Promise { // The warm fixture keeps a reusable sandbox and a warm runner between turns. @@ -69,13 +75,9 @@ async function createDaytonaEnvironment(input: { api: ChaosApi; companyId: strin const payload = fixture.buildEnvironment({ secretRefs: input.secretRefs, daytonaImage: input.image, executionId: input.waveId }); const shape = chaosDaytonaShape(input.cpu); // A user's default environment: the agent decides the lifecycle, and nothing pins lease reuse. - if (input.inherit) { - const config = payload.config as Record; - delete config.runnerLifecycleMode; - delete config.reuseLease; - } + const baseConfig = input.inherit ? inheritingEnvironmentConfig(payload.config as Record) : payload.config as Record; const environment = await input.api.post<{ id: string; driver: string }>(`/api/companies/${input.companyId}/environments`, - { ...payload, config: { ...(payload.config as Record), ...shape, ...(input.reuseLease ? { reuseLease: true } : {}) } }); + { ...payload, config: { ...baseConfig, ...shape, ...(input.reuseLease ? { reuseLease: true } : {}) } }); input.timeline.add("seed_environment", { environmentId: environment.id, driver: environment.driver, image: input.image, ...shape, reuseLease: input.reuseLease }); return environment.id; diff --git a/tests/restart-chaos/ttft-baseline-stats.test.ts b/tests/restart-chaos/ttft-baseline-stats.test.ts index 2a1b7173b96..e5f04482c6d 100644 --- a/tests/restart-chaos/ttft-baseline-stats.test.ts +++ b/tests/restart-chaos/ttft-baseline-stats.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { - baselineRows, distribution, openShellExecutions, percentile, rankPreTurnSteps, recoverySamples, runMetrics, runWarmth, + baselineRows, daytonaBaselineHolders, distribution, parseBaselineEnvironment, openShellExecutions, percentile, rankPreTurnSteps, recoverySamples, runMetrics, runWarmth, type BaselineEvent, type BaselineRun, } from "./ttft-baseline-stats.js"; @@ -77,6 +77,31 @@ describe("ttft baseline statistics", () => { ttft: { n: 2, p50: 6_000, p95: 12_000 } }); }); + it("leaves post-restart and verification runs out of the baseline rows", () => { + const rows = baselineRows([coldRun("a"), run("p", "postfault", [span("task.time_to_first_token", 0, 20_000)]), + run("v", "verify", [span("task.time_to_first_token", 0, 30_000)])]); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ warmth: "cold", ttft: { n: 1, p50: 6_000 } }); + }); + + it("counts held Daytona sandboxes per concurrent task, or per sampled task when they stay warm", () => { + const phases = new Set(["coldwarm", "midturn", "restart"]); + expect(daytonaBaselineHolders({ adapters: 3, samples: 5, phases, restarts: 2, retainsSandboxes: false })).toBe(3); + expect(daytonaBaselineHolders({ adapters: 3, samples: 5, phases, restarts: 2, retainsSandboxes: true })).toBe(60); + expect(daytonaBaselineHolders({ adapters: 2, samples: 0, phases: new Set(["postfault"]), restarts: 0, retainsSandboxes: true })).toBe(2); + }); + + it("rejects contradictory or out-of-range environment options", () => { + const base = { env: "daytona", image: "img", cpu: "4", warm: false, inherit: false }; + expect(parseBaselineEnvironment(base)).toEqual({ env: "daytona", cpu: 4 }); + expect(parseBaselineEnvironment({ ...base, env: "local", image: undefined })).toMatchObject({ env: "local" }); + expect(() => parseBaselineEnvironment({ ...base, env: "dayton" })).toThrow(/local or daytona/); + expect(() => parseBaselineEnvironment({ ...base, image: undefined })).toThrow(/needs --daytona-image/); + for (const cpu of ["0", "5", "abc", "2.5"]) expect(() => parseBaselineEnvironment({ ...base, cpu })).toThrow(/1 to 4/); + expect(() => parseBaselineEnvironment({ ...base, warm: true, inherit: true })).toThrow(/cannot be combined/); + expect(() => parseBaselineEnvironment({ ...base, env: "local", image: undefined, warm: true })).toThrow(/only to --env daytona/); + }); + it("ranks pre-turn steps by p95 without cumulative wrappers", () => { const ranked = rankPreTurnSteps([coldRun("a"), coldRun("b", 2)]); expect(ranked[0]).toMatchObject({ step: "runner.process.launch", n: 2, p95: 3_000 }); diff --git a/tests/restart-chaos/ttft-baseline-stats.ts b/tests/restart-chaos/ttft-baseline-stats.ts index 47caa34ae82..69b89ae867c 100644 --- a/tests/restart-chaos/ttft-baseline-stats.ts +++ b/tests/restart-chaos/ttft-baseline-stats.ts @@ -235,3 +235,31 @@ export function openShellExecutions(events: readonly BaselineEvent[], marker?: s } return [...open.entries()].map(([executionId, entry]) => ({ executionId, matched: entry.matched, unlabelled: !entry.labelled })); } + +/** Daytona sandboxes a TTFT baseline can hold at once, for the vCPU budget check. */ +export function daytonaBaselineHolders(input: { adapters: number; samples: number; phases: ReadonlySet; restarts: number; + retainsSandboxes: boolean }): number { + // The default fixture releases each sandbox after its turn, so only the concurrent tasks hold one. + if (!input.retainsSandboxes) return input.adapters; + // A warm sandbox outlives its task for the idle window and no other task reuses it. + const perTaskPhases = ["coldwarm", "midturn", "verify"].filter(phase => input.phases.has(phase)).length + + (input.phases.has("restart") ? Math.max(1, input.restarts) : 0); + return input.adapters * Math.max(1, input.samples) * Math.max(1, perTaskPhases); +} + +/** Validates the TTFT driver's environment options; returns the environment. */ +export function parseBaselineEnvironment(input: { env: string | undefined; image: string | undefined; cpu: string | undefined; + warm: boolean; inherit: boolean }): { env: "local" | "daytona"; cpu: number } { + const env = input.env ?? "local"; + if (env !== "local" && env !== "daytona") throw new Error(`--env must be local or daytona, got ${env}`); + const cpu = Number(input.cpu ?? "4"); + if (env === "local") { + if (input.warm || input.inherit || input.image) throw new Error("--daytona-* options apply only to --env daytona"); + return { env, cpu }; + } + if (!input.image) throw new Error("--env daytona needs --daytona-image"); + if (!Number.isInteger(cpu) || cpu < 1 || cpu > 4) throw new Error(`--daytona-cpu must be an integer from 1 to 4, got ${input.cpu}`); + // The warm fixture pins its lifecycle; inherit removes the pins. The two contradict. + if (input.warm && input.inherit) throw new Error("--daytona-warm and --daytona-inherit cannot be combined"); + return { env, cpu }; +} diff --git a/tests/restart-chaos/ttft-baseline.ts b/tests/restart-chaos/ttft-baseline.ts index 28e29aa9535..35422c604a6 100644 --- a/tests/restart-chaos/ttft-baseline.ts +++ b/tests/restart-chaos/ttft-baseline.ts @@ -30,7 +30,7 @@ import { DaytonaWaveSweeper } from "./daytona-sweep.js"; import { Timeline } from "./timeline.js"; import type { AdapterId, FaultKind } from "./types.js"; import { - baselineRows, rankPreTurnSteps, recoverySamples, distribution, openShellExecutions, + baselineRows, rankPreTurnSteps, recoverySamples, distribution, openShellExecutions, daytonaBaselineHolders, parseBaselineEnvironment, type BaselineEvent, type BaselineFault, type BaselineRun, } from "./ttft-baseline-stats.js"; @@ -269,20 +269,12 @@ async function main() { const requested = values.adapters!.split(",").map(value => value.trim()) as AdapterId[]; const samples = Math.max(1, Number(values.samples)); const restarts = Math.max(0, Number(values.restarts)); - const env = values.env === "daytona" ? "daytona" as const : "local" as const; - if (env === "daytona" && !values["daytona-image"]) throw new Error("--env daytona needs --daytona-image"); + const { env, cpu } = parseBaselineEnvironment({ env: values.env, image: values["daytona-image"], cpu: values["daytona-cpu"], + warm: values["daytona-warm"] === true, inherit: values["daytona-inherit"] === true }); if (env === "daytona") { - // A warm sandbox (the warm fixture, or the inheriting default) outlives its task for - // the idle window and no other task reuses it, so count one per sampled task. const budget = Number(process.env.RESTART_CHAOS_DAYTONA_VCPU_BUDGET ?? DAYTONA_VCPU_BUDGET); - const perTaskPhases = ["coldwarm", "midturn", "verify"].filter(phase => phases.has(phase)).length - + (phases.has("restart") ? Math.max(1, Number(values.restarts)) : 0); - const cpu = Number(values["daytona-cpu"]); - // The default fixture releases each sandbox after its turn; only a warm one is retained. const retainsSandboxes = values["daytona-warm"] === true || values["daytona-inherit"] === true; - const holders = retainsSandboxes - ? requested.length * Math.max(1, Number(values.samples)) * Math.max(1, perTaskPhases) - : requested.length; + const holders = daytonaBaselineHolders({ adapters: requested.length, samples, phases, restarts, retainsSandboxes }); if (holders * cpu > budget) { throw new Error(`${holders} ${retainsSandboxes ? "sampled tasks (warm sandboxes outlive their task)" : "concurrent tasks"} × --daytona-cpu ${cpu} vCPU can be held at once, ` + `exceeding the Daytona vCPU budget of ${budget}. Lower --samples, --adapters or --daytona-cpu (RESTART_CHAOS_DAYTONA_VCPU_BUDGET sets the budget).`); @@ -360,7 +352,7 @@ async function main() { const api = new ChaosApi(supervisor.baseUrl); const repositoryPath = values["workspace-repo"] ? path.resolve(values["workspace-repo"]) : undefined; projectScopedTasks = Boolean(repositoryPath); - const lab = await seedChaosLab({ api, env, daytonaImage: values["daytona-image"], daytonaCpu: Number(values["daytona-cpu"]), + const lab = await seedChaosLab({ api, env, daytonaImage: values["daytona-image"], daytonaCpu: cpu, daytonaWarm: values["daytona-warm"], daytonaInherit: values["daytona-inherit"], adapters: requested, concurrency: requested.length, budgetCents: 3000, masks: ["rc06"], fixtureRoot: supervisor.paths.temporaryRoot, waveId, timeline, adapterConfigOverrides, repositoryPath, @@ -461,8 +453,10 @@ async function main() { // First turn after an idle control-plane restart: K1 while nothing runs, // then one more follow-up on each task the cold/warm phase left at rest. // The fault must be idle: no timed-out sample may still be running. + // A failed runs read counts as not at rest: the fault must never land on a live turn. const allAtRest = timedOutIssues.size === 0 && (await Promise.all(tasks.map(async task => - (await runsFor(api, task.issueId)).every(run => !ACTIVE_RUN_STATUSES.has(String(run.status)))))).every(Boolean); + (await api.get(`/api/issues/${task.issueId}/runs`, 10_000).catch(() => null)) + ?.every(run => !ACTIVE_RUN_STATUSES.has(String(run.status))) ?? false))).every(Boolean); if (phases.has("postfault") && followUpTasks.size && !allAtRest) { timeline.add("baseline_postfault_skipped", { reason: "work_not_at_rest", timedOut: timedOutIssues.size }); } From b1b644ff0a19793bffcfe271e650440ad47b185d Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 8 Nov 2026 08:41:24 -0600 Subject: [PATCH 3/3] test(restart-chaos): count inherited Daytona sandboxes as warm only when the agent's lifecycle allows it; bind verify tasks to the project Co-Authored-By: Claude Opus 5.5 --- tests/restart-chaos/ttft-baseline-stats.test.ts | 11 ++++++++++- tests/restart-chaos/ttft-baseline-stats.ts | 11 +++++++++++ tests/restart-chaos/ttft-baseline.ts | 16 +++++++++------- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/tests/restart-chaos/ttft-baseline-stats.test.ts b/tests/restart-chaos/ttft-baseline-stats.test.ts index e5f04482c6d..0c5aa9b3ab6 100644 --- a/tests/restart-chaos/ttft-baseline-stats.test.ts +++ b/tests/restart-chaos/ttft-baseline-stats.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { - baselineRows, daytonaBaselineHolders, distribution, parseBaselineEnvironment, openShellExecutions, percentile, rankPreTurnSteps, recoverySamples, runMetrics, runWarmth, + baselineRows, daytonaBaselineHolders, daytonaRetainsSandboxes, distribution, parseBaselineEnvironment, openShellExecutions, percentile, rankPreTurnSteps, recoverySamples, runMetrics, runWarmth, type BaselineEvent, type BaselineRun, } from "./ttft-baseline-stats.js"; @@ -91,6 +91,15 @@ describe("ttft baseline statistics", () => { expect(daytonaBaselineHolders({ adapters: 2, samples: 0, phases: new Set(["postfault"]), restarts: 0, retainsSandboxes: true })).toBe(2); }); + it("treats an inheriting environment as warm only when the agent's lifecycle is not pinned per-turn", () => { + expect(daytonaRetainsSandboxes({ warmEnvironment: true, inheritEnvironment: false, lifecycle: undefined })).toBe(true); + expect(daytonaRetainsSandboxes({ warmEnvironment: false, inheritEnvironment: true, lifecycle: undefined })).toBe(false); + expect(daytonaRetainsSandboxes({ warmEnvironment: false, inheritEnvironment: true, lifecycle: "per_turn" })).toBe(false); + expect(daytonaRetainsSandboxes({ warmEnvironment: false, inheritEnvironment: true, lifecycle: "default" })).toBe(true); + expect(daytonaRetainsSandboxes({ warmEnvironment: false, inheritEnvironment: true, lifecycle: "warm" })).toBe(true); + expect(daytonaRetainsSandboxes({ warmEnvironment: false, inheritEnvironment: false, lifecycle: "warm" })).toBe(false); + }); + it("rejects contradictory or out-of-range environment options", () => { const base = { env: "daytona", image: "img", cpu: "4", warm: false, inherit: false }; expect(parseBaselineEnvironment(base)).toEqual({ env: "daytona", cpu: 4 }); diff --git a/tests/restart-chaos/ttft-baseline-stats.ts b/tests/restart-chaos/ttft-baseline-stats.ts index 69b89ae867c..522701fbfb8 100644 --- a/tests/restart-chaos/ttft-baseline-stats.ts +++ b/tests/restart-chaos/ttft-baseline-stats.ts @@ -236,6 +236,17 @@ export function openShellExecutions(events: readonly BaselineEvent[], marker?: s return [...open.entries()].map(([executionId, entry]) => ({ executionId, matched: entry.matched, unlabelled: !entry.labelled })); } +/** + * Whether finished tasks keep their Daytona sandbox for the idle window. The warm + * fixture always does. An inheriting environment defers to the agent, whose catalog + * profile pins per_turn unless --lifecycle warm or default (the product's warm + * default on a remote sandbox) replaces it. + */ +export function daytonaRetainsSandboxes(input: { warmEnvironment: boolean; inheritEnvironment: boolean; lifecycle: string | undefined }): boolean { + if (input.warmEnvironment) return true; + return input.inheritEnvironment && (input.lifecycle === "warm" || input.lifecycle === "default"); +} + /** Daytona sandboxes a TTFT baseline can hold at once, for the vCPU budget check. */ export function daytonaBaselineHolders(input: { adapters: number; samples: number; phases: ReadonlySet; restarts: number; retainsSandboxes: boolean }): number { diff --git a/tests/restart-chaos/ttft-baseline.ts b/tests/restart-chaos/ttft-baseline.ts index 35422c604a6..5e1f4233ec5 100644 --- a/tests/restart-chaos/ttft-baseline.ts +++ b/tests/restart-chaos/ttft-baseline.ts @@ -30,7 +30,7 @@ import { DaytonaWaveSweeper } from "./daytona-sweep.js"; import { Timeline } from "./timeline.js"; import type { AdapterId, FaultKind } from "./types.js"; import { - baselineRows, rankPreTurnSteps, recoverySamples, distribution, openShellExecutions, daytonaBaselineHolders, parseBaselineEnvironment, + baselineRows, rankPreTurnSteps, recoverySamples, distribution, openShellExecutions, daytonaBaselineHolders, daytonaRetainsSandboxes, parseBaselineEnvironment, type BaselineEvent, type BaselineFault, type BaselineRun, } from "./ttft-baseline-stats.js"; @@ -269,11 +269,16 @@ async function main() { const requested = values.adapters!.split(",").map(value => value.trim()) as AdapterId[]; const samples = Math.max(1, Number(values.samples)); const restarts = Math.max(0, Number(values.restarts)); + const lifecycle = values.lifecycle; + if (lifecycle !== undefined && lifecycle !== "per_turn" && lifecycle !== "warm" && lifecycle !== "default") { + throw new Error("--lifecycle must be per_turn, warm or default"); + } const { env, cpu } = parseBaselineEnvironment({ env: values.env, image: values["daytona-image"], cpu: values["daytona-cpu"], warm: values["daytona-warm"] === true, inherit: values["daytona-inherit"] === true }); if (env === "daytona") { const budget = Number(process.env.RESTART_CHAOS_DAYTONA_VCPU_BUDGET ?? DAYTONA_VCPU_BUDGET); - const retainsSandboxes = values["daytona-warm"] === true || values["daytona-inherit"] === true; + const retainsSandboxes = daytonaRetainsSandboxes({ warmEnvironment: values["daytona-warm"] === true, + inheritEnvironment: values["daytona-inherit"] === true, lifecycle }); const holders = daytonaBaselineHolders({ adapters: requested.length, samples, phases, restarts, retainsSandboxes }); if (holders * cpu > budget) { throw new Error(`${holders} ${retainsSandboxes ? "sampled tasks (warm sandboxes outlive their task)" : "concurrent tasks"} × --daytona-cpu ${cpu} vCPU can be held at once, ` @@ -290,10 +295,6 @@ async function main() { if (midturnFault && midturnMode !== "steer") throw new Error("--midturn-fault applies to --midturn-mode steer"); const verifyFault = values["verify-fault"] as FaultKind | undefined; if (verifyFault !== undefined && !["K1", "K2", "K6"].includes(verifyFault)) throw new Error("--verify-fault must be K1, K2 or K6"); - const lifecycle = values.lifecycle; - if (lifecycle !== undefined && lifecycle !== "per_turn" && lifecycle !== "warm" && lifecycle !== "default") { - throw new Error("--lifecycle must be per_turn, warm or default"); - } // default removes the catalog's pinned per_turn so the server resolves the // product default; absent keeps the catalog setting. const adapterConfigOverrides = lifecycle === "warm" @@ -409,7 +410,8 @@ async function main() { const hired = await api.post(`/api/companies/${lab.companyId}/agents`, { ...source.body, name: `${source.name} verify ${nonce}` }); const agentId = String(hired.id); const issue = await api.post(`/api/companies/${lab.companyId}/issues`, { - title: `TTFT verify ${nonce}`, description: coldTask(nonce), status: "todo", priority: "medium", assigneeAgentId: agentId }); + title: `TTFT verify ${nonce}`, description: coldTask(nonce), status: "todo", priority: "medium", assigneeAgentId: agentId, + ...(projectScopedTasks ? { projectId: lab.projectId } : {}) }); const issueId = String(issue.id); tasks.push({ issueId, adapter, phase: "verify" }); let fault: Awaited> | null = null;