test(restart-chaos): measure TTFT in a real repository workspace, after an idle restart, and on Daytona

- --workspace-repo seeds a real repository as the project's primary workspace;
  tasks then run in its isolated git worktree.
- postfault phase: the first follow-up after an idle K1, reported on its own.
- --env daytona with --daytona-image/--daytona-cpu, --daytona-warm (reusable
  warm fixture) and --daytona-inherit (a user's default environment); every
  sandbox is released before the server stops, on signals too.
- Per-run workspace operation durations; chaos main.ts takes the same
  Daytona fixture options.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-08 08:17:24 -06:00
1 parent 96badeaa42
commit 5d39e40d6f
7 files changed
+218 -20

No files matched your search

+13 -3
View File
@@ -84,6 +84,9 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC
"daytona-image": { type: "string" },
"daytona-cpu": { type: "string" },
"daytona-reuse-lease": { type: "boolean" },
// The reusable warm Daytona fixture, or an environment that inherits the lifecycle (a user's default).
"daytona-warm": { type: "boolean" },
"daytona-inherit": { type: "boolean" },
keep: { type: "boolean" },
"rc06-masked": { type: "boolean" },
"storm-minutes": { type: "string" },
@@ -96,7 +99,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC
const pollMs = integer(values["poll-ms"], "poll-ms", 3_000, 2_000, 5_000);
const budgetCents = integer(values["budget-cents"], "budget-cents", 3_000, 1, 100_000);
if (values.replay !== undefined) {
for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "daytona-reuse-lease", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point"] as const) {
for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "daytona-reuse-lease", "daytona-warm", "daytona-inherit", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point"] as const) {
if (values[key] !== undefined) throw new Error(`--replay reproduces the saved plan; do not combine it with --${key}`);
}
return { mode: "replay", ref: parseReplayRef(values.replay), keep,
@@ -142,7 +145,8 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC
budgetCents,
pollMs,
...(env === "daytona" ? { daytonaImage, daytonaCpu: integer(values["daytona-cpu"], "daytona-cpu", DAYTONA_DEFAULT_CPU, 1, 4),
...(values["daytona-reuse-lease"] ? { daytonaReuseLease: true } : {}) } : {}),
...(values["daytona-reuse-lease"] ? { daytonaReuseLease: true } : {}),
...(values["daytona-warm"] ? { daytonaWarm: true } : {}), ...(values["daytona-inherit"] ? { daytonaInherit: true } : {}) } : {}),
...(values["rc06-masked"] ? { masks: ["rc06"] } : {}),
...(lifecycle ? { lifecycle } : {}),
// Seeded K4 hold upper end; the controller-outage cap is 600 s (#19797).
@@ -154,7 +158,13 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC
if (env !== "daytona" && values["daytona-reuse-lease"]) throw new Error("--daytona-reuse-lease applies only to --env daytona");
if (options.daytonaCpu !== undefined) {
const budget = integer(process.env.RESTART_CHAOS_DAYTONA_VCPU_BUDGET, "RESTART_CHAOS_DAYTONA_VCPU_BUDGET", DAYTONA_VCPU_BUDGET, 1, 1_000);
if (options.concurrency * options.daytonaCpu > budget) {
// A warm sandbox outlives its task for the idle window, and a new task cannot reuse
// another task's sandbox, so with --daytona-warm every task can hold one at once.
const holders = options.daytonaWarm ? Math.max(options.concurrency, options.tasks) : options.concurrency;
if (holders * options.daytonaCpu > budget) {
if (options.daytonaWarm) throw new Error(`--daytona-warm keeps each task's sandbox for the idle window: ${holders} tasks × `
+ `--daytona-cpu ${options.daytonaCpu} vCPU exceeds the Daytona vCPU budget of ${budget}. Lower --tasks or --daytona-cpu `
+ "(RESTART_CHAOS_DAYTONA_VCPU_BUDGET sets the budget).");
throw new Error(`--concurrency ${options.concurrency} × --daytona-cpu ${options.daytonaCpu} vCPU exceeds the Daytona vCPU budget of ${budget} `
+ "for chaos tasks. Lower one of them (RESTART_CHAOS_DAYTONA_VCPU_BUDGET sets the budget).");
}
+51
View File
@@ -0,0 +1,51 @@
import { spawnSync } from "node:child_process";
import { writeFileSync } from "node:fs";
import path from "node:path";
import { assertControllerMatchesImage, defaultDaytonaImageCacheRoot, prepareDaytonaImageArtifacts } from "./daytona-image.js";
import { PLUGIN_BUILD_STAMP, pluginBuildState } from "./plugin-build.js";
import type { Timeline } from "./timeline.js";
function run(command: string, args: string[], cwd: string, label: string) {
const result = spawnSync(command, args, { cwd, stdio: "inherit" });
if (result.status !== 0) throw new Error(`${label} failed (${command} ${args.join(" ")})`);
}
/**
* Mirrors restart-chaos main.ts's Daytona preparation for the TTFT driver.
* The bundled Daytona plugin is outside the root workspace; prepare it as CI
* does, and rebuild it whenever its sources (or the plugin SDK's) changed since
* the last build, so a worktree that moved to a new master tests that master's
* plugin rather than an older build.
*/
export function ensureDaytonaPlugin(repositoryRoot: string, timeline: Timeline) {
const root = path.join(repositoryRoot, "packages/plugins/sandbox-providers/daytona");
// Inputs outside the plugin directory: the linked plugin SDK and the tsconfig chain both packages extend.
const extraInputs = [
...["src", "package.json", "tsconfig.json"].map(file => path.join(repositoryRoot, "packages/plugins/sdk", file)),
...["tsconfig.json", "tsconfig.base.json"].map(file => path.join(repositoryRoot, file)),
];
const state = pluginBuildState(root, extraInputs);
timeline.add("daytona_plugin_build_state", { reason: state.reason, inputHash: state.inputHash });
if (!state.needsBuild) return;
run("pnpm", ["install", "--ignore-workspace", "--frozen-lockfile", "--ignore-scripts"], root, "Daytona plugin install");
run(process.execPath, [path.join(repositoryRoot, "scripts/link-plugin-dev-sdk.mjs")], repositoryRoot, "Plugin SDK link");
run("pnpm", ["--dir", root, "build"], repositoryRoot, "Daytona plugin build");
writeFileSync(path.join(root, PLUGIN_BUILD_STAMP), `${state.inputHash}\n`);
timeline.add("daytona_plugin_built", { inputHash: state.inputHash });
}
/**
* Controller environment for `--env daytona`: extract the image's Linux runnerd
* and provider pack (cached per image digest, no Docker), refuse a controller
* whose runner source differs from the image, and build the Daytona plugin.
*/
export async function prepareDaytonaController(input: { image: string; repositoryRoot: string; timeline: Timeline }):
Promise<Record<string, string>> {
const artifacts = await prepareDaytonaImageArtifacts({ image: input.image, cacheRoot: defaultDaytonaImageCacheRoot(),
log: line => console.info(`[daytona-image] ${line}`) });
const { localContentId } = await assertControllerMatchesImage(artifacts, input.repositoryRoot);
input.timeline.add("daytona_image_matched", { image: input.image, contentId: artifacts.contentId, localContentId, sourceRevision: artifacts.sourceRevision,
runnerdSha256: artifacts.runnerdSha256, providerPackDigest: artifacts.providerPackDigest, platformManifestDigest: artifacts.platformManifestDigest });
ensureDaytonaPlugin(input.repositoryRoot, input.timeline);
return { PAPERCLIP_RUNNER_REMOTE_BINARY_PATH: artifacts.runnerdPath, PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH: artifacts.providerPackPath };
}
+1
View File
@@ -207,6 +207,7 @@ async function main(): Promise<number> {
const api = new ChaosApi(supervisor.baseUrl);
const lab = await seedChaosLab({ api, env: plan.options.env, daytonaImage: plan.options.daytonaImage, daytonaCpu: plan.options.daytonaCpu,
daytonaReuseLease: plan.options.daytonaReuseLease,
daytonaWarm: plan.options.daytonaWarm, daytonaInherit: plan.options.daytonaInherit,
adapters: [...new Set(plan.tasks.map(task => task.adapter))],
concurrency: plan.options.concurrency, budgetCents: plan.options.budgetCents, masks: plan.options.masks,
fixtureRoot: path.join(supervisor.paths.temporaryRoot, "fixtures"), waveId: plan.waveId, timeline,
+19 -5
View File
@@ -2,7 +2,7 @@ import { execFileSync } from "node:child_process";
import { chmod, mkdir, writeFile } from "node:fs/promises";
import path from "node:path";
import type { RunnerApi } from "../runner-e2e/api.js";
import { runnerEnvironments, runnerProfiles } from "../runner-e2e/catalog.js";
import { daytonaWarmEnvironment, runnerEnvironments, runnerProfiles } from "../runner-e2e/catalog.js";
import { installFixtureDaytonaProvider, waitForFixtureDaytonaProvider } from "../runner-e2e/live-fixtures.js";
import type { MatrixExecution, SecretReference, SecretReferenceMap } from "../runner-e2e/types.js";
import type { ChaosApi } from "./api.js";
@@ -60,13 +60,20 @@ export function chaosDaytonaShape(cpu: number) {
}
async function createDaytonaEnvironment(input: { api: ChaosApi; companyId: string; image: string; cpu: number; reuseLease: boolean; waveId: string;
secretRefs: SecretReferenceMap; timeline: Timeline }): Promise<string> {
const fixture = runnerEnvironments.find(candidate => candidate.id === "daytona");
secretRefs: SecretReferenceMap; timeline: Timeline; warm?: boolean; inherit?: boolean }): Promise<string> {
// The warm fixture keeps a reusable sandbox and a warm runner between turns.
const fixture = input.warm ? daytonaWarmEnvironment : runnerEnvironments.find(candidate => candidate.id === "daytona");
if (!fixture) throw new Error("runner-e2e catalog has no daytona environment fixture");
const view = runnerApiView(input.api);
await waitForFixtureDaytonaProvider(view, { companyId: input.companyId, deadlineAt: Date.now() + 120_000 });
const payload = fixture.buildEnvironment({ secretRefs: input.secretRefs, daytonaImage: input.image, executionId: input.waveId });
const shape = chaosDaytonaShape(input.cpu);
// A user's default environment: the agent decides the lifecycle, and nothing pins lease reuse.
if (input.inherit) {
const config = payload.config as Record<string, unknown>;
delete config.runnerLifecycleMode;
delete config.reuseLease;
}
const environment = await input.api.post<{ id: string; driver: string }>(`/api/companies/${input.companyId}/environments`,
{ ...payload, config: { ...(payload.config as Record<string, unknown>), ...shape, ...(input.reuseLease ? { reuseLease: true } : {}) } });
input.timeline.add("seed_environment", { environmentId: environment.id, driver: environment.driver, image: input.image, ...shape,
@@ -120,6 +127,12 @@ export async function seedChaosLab(input: {
timeline: Timeline;
/** Merged into each agent's adapterConfig (for example a runner lifecycle mode). */
adapterConfigOverrides?: Record<string, unknown>;
/** Use the reusable warm Daytona environment fixture (reuseLease, warm runner). */
daytonaWarm?: boolean;
/** Use a Daytona environment that inherits the lifecycle and leaves lease reuse unset, as a user's default does. */
daytonaInherit?: boolean;
/** A real repository to use as the project's primary workspace instead of the tiny fixture. */
repositoryPath?: string;
/** Called as soon as the wave owns external resources that need cleanup. */
onEnvironment?: (lab: Pick<ChaosLab, "companyId" | "environmentId" | "environmentKind">) => void;
}): Promise<ChaosLab> {
@@ -130,7 +143,8 @@ export async function seedChaosLab(input: {
await installFixtureDaytonaProvider(runnerApiView(input.api), { environment: { id: "daytona" } } as MatrixExecution);
timeline.add("seed_daytona_plugin", {});
}
const fixtureRepositoryPath = await createFixtureRepository(path.join(input.fixtureRoot, "chaos-sandbox"));
const fixtureRepositoryPath = input.repositoryPath
?? await createFixtureRepository(path.join(input.fixtureRoot, "chaos-sandbox"));
const company = await api.post<{ id: string }>("/api/companies", {
name: "Chaos Lab",
description: `Restart chaos harness wave ${input.waveId}`,
@@ -161,7 +175,7 @@ export async function seedChaosLab(input: {
let environmentId: string;
if (input.env === "daytona") {
environmentId = await createDaytonaEnvironment({ api, companyId: company.id, image: input.daytonaImage!, cpu: input.daytonaCpu ?? 4,
reuseLease: input.daytonaReuseLease === true, waveId: input.waveId, secretRefs, timeline });
reuseLease: input.daytonaReuseLease === true, waveId: input.waveId, secretRefs, timeline, warm: input.daytonaWarm, inherit: input.daytonaInherit });
input.onEnvironment?.({ companyId: company.id, environmentId, environmentKind: "daytona" });
if (input.adapters.includes("codex")) await qualifyNativeCodex({ api, companyId: company.id, environmentId, timeline });
} else {
+3 -3
View File
@@ -16,7 +16,7 @@ export interface BaselineRun {
adapter: string;
environment: string;
/** Driver intent: a new task, a follow-up comment on the same task, or a run crossed by a restart. */
phase: "cold" | "warm" | "restart" | "midturn" | "verify";
phase: "cold" | "warm" | "restart" | "midturn" | "verify" | "postfault";
events: BaselineEvent[];
}
@@ -149,8 +149,8 @@ export interface BaselineRow { adapter: string; environment: string; warmth: "co
export function baselineRows(runs: readonly BaselineRun[]): BaselineRow[] {
const groups = new Map<string, { adapter: string; environment: string; warmth: "cold" | "warm"; metrics: RunMetrics[] }>();
for (const run of runs) {
// Restart samples and new-agent verification samples are reported on their own.
if (run.phase === "restart" || run.phase === "verify") continue;
// Restart, post-restart and new-agent verification samples are reported on their own.
if (run.phase === "restart" || run.phase === "verify" || run.phase === "postfault") continue;
const warmth = runWarmth(run);
const key = `${run.adapter}|${run.environment}|${warmth}`;
const group = groups.get(key) ?? { adapter: run.adapter, environment: run.environment, warmth, metrics: [] };
+127 -9
View File
@@ -20,11 +20,13 @@ import path from "node:path";
import { parseArgs } from "node:util";
import { setTimeout as delay } from "node:timers/promises";
import { ChaosApi } from "./api.js";
import { CREDENTIAL_FOR_ADAPTER } from "./config.js";
import { CREDENTIAL_FOR_ADAPTER, DAYTONA_VCPU_BUDGET } from "./config.js";
import { ACTIVE_RUN_STATUSES } from "./grader.js";
import { chaosLifecycleOverrides, seedChaosLab, type ChaosLab } from "./seed.js";
import { ChaosSupervisor } from "./supervisor.js";
import { createDb } from "../../packages/db/src/index.js";
import { prepareDaytonaController } from "./daytona-controller.js";
import { DaytonaWaveSweeper } from "./daytona-sweep.js";
import { Timeline } from "./timeline.js";
import type { AdapterId, FaultKind } from "./types.js";
import {
@@ -88,9 +90,13 @@ async function steerQueuedComment(api: ChaosApi, issueId: string, commentId: str
return { acknowledgedAt: null, disposition };
}
/** Set by --workspace-repo: tasks then run in the project's isolated workspace of that repository. */
let projectScopedTasks = false;
async function createTask(api: ChaosApi, lab: ChaosLab, adapter: AdapterId, title: string, description: string) {
const issue = await api.post<Json>(`/api/companies/${lab.companyId}/issues`, {
title, description, status: "todo", priority: "medium", assigneeAgentId: lab.agents[adapter].id,
...(projectScopedTasks ? { projectId: lab.projectId } : {}),
});
return String(issue.id);
}
@@ -200,6 +206,11 @@ function report(runs: BaselineRun[], faults: BaselineFault[]): string {
lines.push("", "New agent behind harness verification (TTFT counts from run creation; the verification wait precedes it):", "", lines[0]!, lines[1]!);
for (const row of baselineRows(verifyRuns)) lines.push(rowLine(row));
}
const postfault = runs.filter(run => run.phase === "postfault").map(run => ({ ...run, phase: "cold" as const }));
if (postfault.length) {
lines.push("", "First follow-up after an idle control-plane restart (K1):", "", lines[0]!, lines[1]!);
for (const row of baselineRows(postfault)) lines.push(rowLine(row));
}
lines.push("", "| Pre-turn step | n | p50 | p95 | Adapters |", "|---|---|---|---|---|");
for (const step of rankPreTurnSteps(runs).slice(0, 25)) lines.push(`| \`${step.step}\` | ${step.n} | ${ms(step.p50)} | ${ms(step.p95)} | ${step.adapters.join(", ")} |`);
@@ -236,6 +247,17 @@ async function main() {
"midturn-fault": { type: "string" },
// verify phase: fault the control plane while a newly hired agent runs harness verification.
"verify-fault": { type: "string" },
// A real repository (for example a fresh clone of paperclip) as the project's
// primary workspace; every task then runs in its isolated git worktree.
"workspace-repo": { type: "string" },
// local (default) or daytona; daytona needs --daytona-image (the signed Runner image).
env: { type: "string", default: "local" },
"daytona-image": { type: "string" },
"daytona-cpu": { type: "string", default: "4" },
// Use the reusable warm Daytona environment (reuseLease, warm runner) instead of the per-turn one.
"daytona-warm": { type: "boolean", default: false },
// A Daytona environment as a user creates it by default: lifecycle inherited, lease reuse unset.
"daytona-inherit": { type: "boolean", default: false },
"idle-ms": { type: "string", default: "600000" },
} });
if (values.report) {
@@ -247,7 +269,27 @@ async function main() {
const requested = values.adapters!.split(",").map(value => value.trim()) as AdapterId[];
const samples = Math.max(1, Number(values.samples));
const restarts = Math.max(0, Number(values.restarts));
const missing = requested.map(adapter => CREDENTIAL_FOR_ADAPTER[adapter]).filter(key => !process.env[key]);
const env = values.env === "daytona" ? "daytona" as const : "local" as const;
if (env === "daytona" && !values["daytona-image"]) throw new Error("--env daytona needs --daytona-image");
if (env === "daytona") {
// A warm sandbox (the warm fixture, or the inheriting default) outlives its task for
// the idle window and no other task reuses it, so count one per sampled task.
const budget = Number(process.env.RESTART_CHAOS_DAYTONA_VCPU_BUDGET ?? DAYTONA_VCPU_BUDGET);
const perTaskPhases = ["coldwarm", "midturn", "verify"].filter(phase => phases.has(phase)).length
+ (phases.has("restart") ? Math.max(1, Number(values.restarts)) : 0);
const cpu = Number(values["daytona-cpu"]);
// The default fixture releases each sandbox after its turn; only a warm one is retained.
const retainsSandboxes = values["daytona-warm"] === true || values["daytona-inherit"] === true;
const holders = retainsSandboxes
? requested.length * Math.max(1, Number(values.samples)) * Math.max(1, perTaskPhases)
: requested.length;
if (holders * cpu > budget) {
throw new Error(`${holders} ${retainsSandboxes ? "sampled tasks (warm sandboxes outlive their task)" : "concurrent tasks"} × --daytona-cpu ${cpu} vCPU can be held at once, `
+ `exceeding the Daytona vCPU budget of ${budget}. Lower --samples, --adapters or --daytona-cpu (RESTART_CHAOS_DAYTONA_VCPU_BUDGET sets the budget).`);
}
}
const missing = [...requested.map(adapter => CREDENTIAL_FOR_ADAPTER[adapter]), ...(env === "daytona" ? ["DAYTONA_API_KEY"] : [])]
.filter(key => !process.env[key]);
if (missing.length) throw new Error(`Missing ${missing.join(", ")}; run: set -a; source ~/.secrets; set +a`);
const midturnMode = values["midturn-mode"];
if (midturnMode !== "queue" && midturnMode !== "steer") throw new Error("--midturn-mode must be queue or steer");
@@ -276,10 +318,34 @@ async function main() {
const agentLifecycles: Record<string, string> = {};
const midturn: Json[] = [];
const timedOutIssues = new Set<string>();
let sweeper: DaytonaWaveSweeper | null = null;
const workspaceOperations: Array<{ runId: string; adapter: string; phase: string; operation: string; status: string;
durationMs: number | null; command: string | null }> = [];
const followUpTasks = new Map<AdapterId, { issueId: string; nonce: string; runs: number }>();
// One shutdown path for normal completion and for SIGINT/SIGTERM: the server
// runs in a detached process group and the embedded Postgres has no exit hook.
let stopping: Promise<void> | null = null;
const shutdown = () => (stopping ??= supervisor.stop().catch(error => {
// Daytona resources are released while the server still runs, before it stops;
// the same path serves normal completion and SIGINT/SIGTERM.
const releaseDaytona = async () => {
if (!sweeper) return;
const owned = sweeper as DaytonaWaveSweeper;
sweeper = null;
try {
const cleanup = await owned.cleanUp();
timeline.add("daytona_cleanup", cleanup);
await writeFile(path.join(outDir, "sandbox-cleanup.json"), `${JSON.stringify(cleanup, null, 2)}\n`, { mode: 0o600 });
if (cleanup.remainingSandboxes.length || cleanup.errors.length || cleanup.environmentDeleteBlocked || !cleanup.environmentDeleted) {
process.stderr.write(`Daytona cleanup incomplete: remaining ${cleanup.remainingSandboxes.join(", ") || "none"}; `
+ `environment deleted ${cleanup.environmentDeleted}${cleanup.environmentDeleteBlocked ? ` (blocked: ${String(cleanup.environmentDeleteBlocked)})` : ""}; ${cleanup.errors.join("; ")}\n`);
process.exitCode = 1;
}
} catch (error) {
timeline.add("daytona_cleanup_failed", { error: String(error) });
process.exitCode = 1;
}
};
const shutdown = () => (stopping ??= releaseDaytona().then(() => supervisor.stop()).catch(error => {
process.stderr.write(`supervisor stop: ${String(error)}\n`);
}));
const onSignal = (signal: NodeJS.Signals) => {
@@ -289,10 +355,18 @@ async function main() {
process.once("SIGINT", onSignal);
process.once("SIGTERM", onSignal);
try {
await supervisor.start();
await supervisor.start(env === "daytona"
? await prepareDaytonaController({ image: values["daytona-image"]!, repositoryRoot, timeline }) : {});
const api = new ChaosApi(supervisor.baseUrl);
const lab = await seedChaosLab({ api, env: "local", adapters: requested, concurrency: requested.length, budgetCents: 3000,
masks: ["rc06"], fixtureRoot: supervisor.paths.temporaryRoot, waveId, timeline, adapterConfigOverrides });
const repositoryPath = values["workspace-repo"] ? path.resolve(values["workspace-repo"]) : undefined;
projectScopedTasks = Boolean(repositoryPath);
const lab = await seedChaosLab({ api, env, daytonaImage: values["daytona-image"], daytonaCpu: Number(values["daytona-cpu"]),
daytonaWarm: values["daytona-warm"], daytonaInherit: values["daytona-inherit"],
adapters: requested, concurrency: requested.length, budgetCents: 3000,
masks: ["rc06"], fixtureRoot: supervisor.paths.temporaryRoot, waveId, timeline, adapterConfigOverrides, repositoryPath,
onEnvironment: owned => {
sweeper = new DaytonaWaveSweeper({ apiKey: process.env.DAYTONA_API_KEY!, companyId: owned.companyId, environmentId: owned.environmentId, api });
} });
const adapters = await readyAdapters(api, lab, requested, timeline);
if (!adapters.length) throw new Error("No agent passed lifecycle verification");
// Record what each agent actually stores: "unset" means the server resolves
@@ -328,6 +402,7 @@ async function main() {
continue;
}
timeline.add("baseline_warm_done", { adapter, issueId, runs: warm.runs.length - cold.runs.length });
followUpTasks.set(adapter, { issueId, nonce, runs: warm.runs.length });
}
}));
@@ -383,6 +458,29 @@ async function main() {
}
}
// First turn after an idle control-plane restart: K1 while nothing runs,
// then one more follow-up on each task the cold/warm phase left at rest.
// The fault must be idle: no timed-out sample may still be running.
const allAtRest = timedOutIssues.size === 0 && (await Promise.all(tasks.map(async task =>
(await runsFor(api, task.issueId)).every(run => !ACTIVE_RUN_STATUSES.has(String(run.status)))))).every(Boolean);
if (phases.has("postfault") && followUpTasks.size && !allAtRest) {
timeline.add("baseline_postfault_skipped", { reason: "work_not_at_rest", timedOut: timedOutIssues.size });
}
if (phases.has("postfault") && followUpTasks.size && allAtRest) {
const receipt = await supervisor.fault("K1", 0, "ttft-baseline-postfault");
faults.push({ kind: "K1", readyAt: receipt.readyAt ?? Date.now(), runIds: [] });
await Promise.all([...followUpTasks.entries()].map(async ([adapter, entry]) => {
await api.post(`/api/issues/${entry.issueId}/comments`, { body: warmFollowUp(`${entry.nonce}-postfault`), reopen: true });
const after = await waitForRest(api, supervisor, entry.issueId, entry.runs + 1);
for (const run of after.runs) if (!runPhase.has(runIdOf(run))) runPhase.set(runIdOf(run), "postfault");
if (after.timedOut) {
timedOutIssues.add(entry.issueId);
timeline.add("baseline_sample_timeout", { adapter, issueId: entry.issueId, phase: "postfault" });
}
else timeline.add("baseline_postfault_done", { adapter, issueId: entry.issueId });
}));
}
// Mid-turn follow-up: a comment posted while the agent is inside a long
// tool call. Records when the comment landed and every run of the task,
// so the wait splits into "rest of the turn" and "hand-off".
@@ -472,15 +570,34 @@ async function main() {
if (timedOutIssues.has(task.issueId)) continue;
for (const run of await runsFor(api, task.issueId)) {
const runId = runIdOf(run);
runs.push({ runId, adapter: task.adapter, environment: "local",
runs.push({ runId, adapter: task.adapter, environment: env,
phase: runPhase.get(runId) ?? task.phase, events: await allEvents(api, runId) });
// Workspace commands (worktree, provision, sync) the run waited on, with their durations.
const operations = await api.get<Json[]>(`/api/heartbeat-runs/${runId}/workspace-operations`, 10_000).catch((error: unknown) => {
// A failed read is recorded as such, never as "no workspace operations".
workspaceOperations.push({ runId, adapter: task.adapter, phase: runPhase.get(runId) ?? task.phase,
operation: "read_failed", status: error instanceof Error ? error.message.slice(0, 160) : String(error), durationMs: null, command: null });
return [] as Json[];
});
for (const operation of operations) workspaceOperations.push({ runId, adapter: task.adapter,
phase: runPhase.get(runId) ?? task.phase, operation: String(operation.phase), status: String(operation.status),
durationMs: operation.startedAt && operation.finishedAt
? Date.parse(String(operation.finishedAt)) - Date.parse(String(operation.startedAt)) : null,
command: typeof operation.command === "string" ? operation.command.slice(0, 120) : null });
}
}
const data = { waveId, lifecycle: lifecycle ?? "catalog", agentLifecycles, adapters, notReady: requested.filter(adapter => !adapters.includes(adapter)),
const data = { workspaceOperations, waveId, lifecycle: lifecycle ?? "catalog", agentLifecycles, adapters, notReady: requested.filter(adapter => !adapters.includes(adapter)),
timedOutTasks: timedOutIssues.size, samples, restarts, midturn, gitSha: process.env.GIT_SHA ?? null, runs, faults };
await writeFile(path.join(outDir, "baseline-data.json"), `${JSON.stringify(data)}\n`, { mode: 0o600 });
await writeFile(path.join(outDir, "baseline-report.md"), report(runs, faults), { mode: 0o600 });
process.stdout.write(report(runs, faults));
if (workspaceOperations.length) {
const table = ["", "| Run phase | Adapter | Workspace operation | Status | Duration | Command |", "|---|---|---|---|---|---|",
...workspaceOperations.slice().sort((a, b) => (b.durationMs ?? 0) - (a.durationMs ?? 0))
.map(op => `| ${op.phase} | ${op.adapter} | ${op.operation} | ${op.status} | ${op.durationMs === null ? "—" : `${(op.durationMs / 1000).toFixed(2)} s`} | ${op.command ? `\`${op.command}\`` : "—"} |`)].join("\n");
process.stdout.write(`${table}\n`);
await writeFile(path.join(outDir, "workspace-operations.md"), `${table}\n`, { mode: 0o600 });
}
if (timedOutIssues.size) process.stdout.write(`\n${timedOutIssues.size} sample(s) timed out and were left out.\n`);
// Every answered mid-turn comment must be answered exactly once.
const wrongReplies = midturn.filter(sample => sample.replyCount !== 1);
@@ -491,7 +608,8 @@ async function main() {
if (wrongReplies.length) process.exitCode = 1;
process.stdout.write(`\nWrote ${outDir}\n`);
} finally {
// Always stop the wrapper-owned embedded Postgres (SysV shared memory is scarce).
// Every Daytona sandbox this run created is archived or destroyed before the server
// stops, then the wrapper-owned embedded Postgres stops (SysV shared memory is scarce).
await shutdown();
process.off("SIGINT", onSignal);
process.off("SIGTERM", onSignal);
+4
View File
@@ -36,6 +36,10 @@ export interface WaveOptions {
daytonaCpu?: number;
/** `--daytona-reuse-lease`: the chaos environment keeps its sandboxes between turns (a per-turn run stops and keeps its sandbox). */
daytonaReuseLease?: boolean;
/** `--daytona-warm`: the reusable warm Daytona environment fixture. */
daytonaWarm?: boolean;
/** `--daytona-inherit`: a Daytona environment that inherits the lifecycle and leaves lease reuse unset. */
daytonaInherit?: boolean;
/** `--k4-hold-max-seconds`: upper end of the seeded K4 hold (default 300 s, cap 600 s). */
k4HoldMaxMs?: number;
/**