From b1b644ff0a19793bffcfe271e650440ad47b185d Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 8 Nov 2026 08:41:24 -0600 Subject: [PATCH] test(restart-chaos): count inherited Daytona sandboxes as warm only when the agent's lifecycle allows it; bind verify tasks to the project Co-Authored-By: Claude Opus 5.5 --- tests/restart-chaos/ttft-baseline-stats.test.ts | 11 ++++++++++- tests/restart-chaos/ttft-baseline-stats.ts | 11 +++++++++++ tests/restart-chaos/ttft-baseline.ts | 16 +++++++++------- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/tests/restart-chaos/ttft-baseline-stats.test.ts b/tests/restart-chaos/ttft-baseline-stats.test.ts index e5f04482c6d..0c5aa9b3ab6 100644 --- a/tests/restart-chaos/ttft-baseline-stats.test.ts +++ b/tests/restart-chaos/ttft-baseline-stats.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { - baselineRows, daytonaBaselineHolders, distribution, parseBaselineEnvironment, openShellExecutions, percentile, rankPreTurnSteps, recoverySamples, runMetrics, runWarmth, + baselineRows, daytonaBaselineHolders, daytonaRetainsSandboxes, distribution, parseBaselineEnvironment, openShellExecutions, percentile, rankPreTurnSteps, recoverySamples, runMetrics, runWarmth, type BaselineEvent, type BaselineRun, } from "./ttft-baseline-stats.js"; @@ -91,6 +91,15 @@ describe("ttft baseline statistics", () => { expect(daytonaBaselineHolders({ adapters: 2, samples: 0, phases: new Set(["postfault"]), restarts: 0, retainsSandboxes: true })).toBe(2); }); + it("treats an inheriting environment as warm only when the agent's lifecycle is not pinned per-turn", () => { + expect(daytonaRetainsSandboxes({ warmEnvironment: true, inheritEnvironment: false, lifecycle: undefined })).toBe(true); + expect(daytonaRetainsSandboxes({ warmEnvironment: false, inheritEnvironment: true, lifecycle: undefined })).toBe(false); + expect(daytonaRetainsSandboxes({ warmEnvironment: false, inheritEnvironment: true, lifecycle: "per_turn" })).toBe(false); + expect(daytonaRetainsSandboxes({ warmEnvironment: false, inheritEnvironment: true, lifecycle: "default" })).toBe(true); + expect(daytonaRetainsSandboxes({ warmEnvironment: false, inheritEnvironment: true, lifecycle: "warm" })).toBe(true); + expect(daytonaRetainsSandboxes({ warmEnvironment: false, inheritEnvironment: false, lifecycle: "warm" })).toBe(false); + }); + it("rejects contradictory or out-of-range environment options", () => { const base = { env: "daytona", image: "img", cpu: "4", warm: false, inherit: false }; expect(parseBaselineEnvironment(base)).toEqual({ env: "daytona", cpu: 4 }); diff --git a/tests/restart-chaos/ttft-baseline-stats.ts b/tests/restart-chaos/ttft-baseline-stats.ts index 69b89ae867c..522701fbfb8 100644 --- a/tests/restart-chaos/ttft-baseline-stats.ts +++ b/tests/restart-chaos/ttft-baseline-stats.ts @@ -236,6 +236,17 @@ export function openShellExecutions(events: readonly BaselineEvent[], marker?: s return [...open.entries()].map(([executionId, entry]) => ({ executionId, matched: entry.matched, unlabelled: !entry.labelled })); } +/** + * Whether finished tasks keep their Daytona sandbox for the idle window. The warm + * fixture always does. An inheriting environment defers to the agent, whose catalog + * profile pins per_turn unless --lifecycle warm or default (the product's warm + * default on a remote sandbox) replaces it. + */ +export function daytonaRetainsSandboxes(input: { warmEnvironment: boolean; inheritEnvironment: boolean; lifecycle: string | undefined }): boolean { + if (input.warmEnvironment) return true; + return input.inheritEnvironment && (input.lifecycle === "warm" || input.lifecycle === "default"); +} + /** Daytona sandboxes a TTFT baseline can hold at once, for the vCPU budget check. */ export function daytonaBaselineHolders(input: { adapters: number; samples: number; phases: ReadonlySet; restarts: number; retainsSandboxes: boolean }): number { diff --git a/tests/restart-chaos/ttft-baseline.ts b/tests/restart-chaos/ttft-baseline.ts index 35422c604a6..5e1f4233ec5 100644 --- a/tests/restart-chaos/ttft-baseline.ts +++ b/tests/restart-chaos/ttft-baseline.ts @@ -30,7 +30,7 @@ import { DaytonaWaveSweeper } from "./daytona-sweep.js"; import { Timeline } from "./timeline.js"; import type { AdapterId, FaultKind } from "./types.js"; import { - baselineRows, rankPreTurnSteps, recoverySamples, distribution, openShellExecutions, daytonaBaselineHolders, parseBaselineEnvironment, + baselineRows, rankPreTurnSteps, recoverySamples, distribution, openShellExecutions, daytonaBaselineHolders, daytonaRetainsSandboxes, parseBaselineEnvironment, type BaselineEvent, type BaselineFault, type BaselineRun, } from "./ttft-baseline-stats.js"; @@ -269,11 +269,16 @@ async function main() { const requested = values.adapters!.split(",").map(value => value.trim()) as AdapterId[]; const samples = Math.max(1, Number(values.samples)); const restarts = Math.max(0, Number(values.restarts)); + const lifecycle = values.lifecycle; + if (lifecycle !== undefined && lifecycle !== "per_turn" && lifecycle !== "warm" && lifecycle !== "default") { + throw new Error("--lifecycle must be per_turn, warm or default"); + } const { env, cpu } = parseBaselineEnvironment({ env: values.env, image: values["daytona-image"], cpu: values["daytona-cpu"], warm: values["daytona-warm"] === true, inherit: values["daytona-inherit"] === true }); if (env === "daytona") { const budget = Number(process.env.RESTART_CHAOS_DAYTONA_VCPU_BUDGET ?? DAYTONA_VCPU_BUDGET); - const retainsSandboxes = values["daytona-warm"] === true || values["daytona-inherit"] === true; + const retainsSandboxes = daytonaRetainsSandboxes({ warmEnvironment: values["daytona-warm"] === true, + inheritEnvironment: values["daytona-inherit"] === true, lifecycle }); const holders = daytonaBaselineHolders({ adapters: requested.length, samples, phases, restarts, retainsSandboxes }); if (holders * cpu > budget) { throw new Error(`${holders} ${retainsSandboxes ? "sampled tasks (warm sandboxes outlive their task)" : "concurrent tasks"} × --daytona-cpu ${cpu} vCPU can be held at once, ` @@ -290,10 +295,6 @@ async function main() { if (midturnFault && midturnMode !== "steer") throw new Error("--midturn-fault applies to --midturn-mode steer"); const verifyFault = values["verify-fault"] as FaultKind | undefined; if (verifyFault !== undefined && !["K1", "K2", "K6"].includes(verifyFault)) throw new Error("--verify-fault must be K1, K2 or K6"); - const lifecycle = values.lifecycle; - if (lifecycle !== undefined && lifecycle !== "per_turn" && lifecycle !== "warm" && lifecycle !== "default") { - throw new Error("--lifecycle must be per_turn, warm or default"); - } // default removes the catalog's pinned per_turn so the server resolves the // product default; absent keeps the catalog setting. const adapterConfigOverrides = lifecycle === "warm" @@ -409,7 +410,8 @@ async function main() { const hired = await api.post(`/api/companies/${lab.companyId}/agents`, { ...source.body, name: `${source.name} verify ${nonce}` }); const agentId = String(hired.id); const issue = await api.post(`/api/companies/${lab.companyId}/issues`, { - title: `TTFT verify ${nonce}`, description: coldTask(nonce), status: "todo", priority: "medium", assigneeAgentId: agentId }); + title: `TTFT verify ${nonce}`, description: coldTask(nonce), status: "todo", priority: "medium", assigneeAgentId: agentId, + ...(projectScopedTasks ? { projectId: lab.projectId } : {}) }); const issueId = String(issue.id); tasks.push({ issueId, adapter, phase: "verify" }); let fault: Awaited> | null = null;