mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 12:37:55 +01:00
test(restart-chaos): measure TTFT in a real repository workspace, after an idle restart, and on Daytona
- --workspace-repo seeds a real repository as the project's primary workspace; tasks then run in its isolated git worktree. - postfault phase: the first follow-up after an idle K1, reported on its own. - --env daytona with --daytona-image/--daytona-cpu, --daytona-warm (reusable warm fixture) and --daytona-inherit (a user's default environment); every sandbox is released before the server stops, on signals too. - Per-run workspace operation durations; chaos main.ts takes the same Daytona fixture options. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
96badeaa42
commit
5d39e40d6f
7 files changed
+218
-20
No files matched your search
@@ -84,6 +84,9 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC
|
||||
"daytona-image": { type: "string" },
|
||||
"daytona-cpu": { type: "string" },
|
||||
"daytona-reuse-lease": { type: "boolean" },
|
||||
// The reusable warm Daytona fixture, or an environment that inherits the lifecycle (a user's default).
|
||||
"daytona-warm": { type: "boolean" },
|
||||
"daytona-inherit": { type: "boolean" },
|
||||
keep: { type: "boolean" },
|
||||
"rc06-masked": { type: "boolean" },
|
||||
"storm-minutes": { type: "string" },
|
||||
@@ -96,7 +99,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC
|
||||
const pollMs = integer(values["poll-ms"], "poll-ms", 3_000, 2_000, 5_000);
|
||||
const budgetCents = integer(values["budget-cents"], "budget-cents", 3_000, 1, 100_000);
|
||||
if (values.replay !== undefined) {
|
||||
for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "daytona-reuse-lease", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point"] as const) {
|
||||
for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "daytona-reuse-lease", "daytona-warm", "daytona-inherit", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point"] as const) {
|
||||
if (values[key] !== undefined) throw new Error(`--replay reproduces the saved plan; do not combine it with --${key}`);
|
||||
}
|
||||
return { mode: "replay", ref: parseReplayRef(values.replay), keep,
|
||||
@@ -142,7 +145,8 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC
|
||||
budgetCents,
|
||||
pollMs,
|
||||
...(env === "daytona" ? { daytonaImage, daytonaCpu: integer(values["daytona-cpu"], "daytona-cpu", DAYTONA_DEFAULT_CPU, 1, 4),
|
||||
...(values["daytona-reuse-lease"] ? { daytonaReuseLease: true } : {}) } : {}),
|
||||
...(values["daytona-reuse-lease"] ? { daytonaReuseLease: true } : {}),
|
||||
...(values["daytona-warm"] ? { daytonaWarm: true } : {}), ...(values["daytona-inherit"] ? { daytonaInherit: true } : {}) } : {}),
|
||||
...(values["rc06-masked"] ? { masks: ["rc06"] } : {}),
|
||||
...(lifecycle ? { lifecycle } : {}),
|
||||
// Seeded K4 hold upper end; the controller-outage cap is 600 s (#19797).
|
||||
@@ -154,7 +158,13 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC
|
||||
if (env !== "daytona" && values["daytona-reuse-lease"]) throw new Error("--daytona-reuse-lease applies only to --env daytona");
|
||||
if (options.daytonaCpu !== undefined) {
|
||||
const budget = integer(process.env.RESTART_CHAOS_DAYTONA_VCPU_BUDGET, "RESTART_CHAOS_DAYTONA_VCPU_BUDGET", DAYTONA_VCPU_BUDGET, 1, 1_000);
|
||||
if (options.concurrency * options.daytonaCpu > budget) {
|
||||
// A warm sandbox outlives its task for the idle window, and a new task cannot reuse
|
||||
// another task's sandbox, so with --daytona-warm every task can hold one at once.
|
||||
const holders = options.daytonaWarm ? Math.max(options.concurrency, options.tasks) : options.concurrency;
|
||||
if (holders * options.daytonaCpu > budget) {
|
||||
if (options.daytonaWarm) throw new Error(`--daytona-warm keeps each task's sandbox for the idle window: ${holders} tasks × `
|
||||
+ `--daytona-cpu ${options.daytonaCpu} vCPU exceeds the Daytona vCPU budget of ${budget}. Lower --tasks or --daytona-cpu `
|
||||
+ "(RESTART_CHAOS_DAYTONA_VCPU_BUDGET sets the budget).");
|
||||
throw new Error(`--concurrency ${options.concurrency} × --daytona-cpu ${options.daytonaCpu} vCPU exceeds the Daytona vCPU budget of ${budget} `
|
||||
+ "for chaos tasks. Lower one of them (RESTART_CHAOS_DAYTONA_VCPU_BUDGET sets the budget).");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { assertControllerMatchesImage, defaultDaytonaImageCacheRoot, prepareDaytonaImageArtifacts } from "./daytona-image.js";
|
||||
import { PLUGIN_BUILD_STAMP, pluginBuildState } from "./plugin-build.js";
|
||||
import type { Timeline } from "./timeline.js";
|
||||
|
||||
function run(command: string, args: string[], cwd: string, label: string) {
|
||||
const result = spawnSync(command, args, { cwd, stdio: "inherit" });
|
||||
if (result.status !== 0) throw new Error(`${label} failed (${command} ${args.join(" ")})`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Mirrors restart-chaos main.ts's Daytona preparation for the TTFT driver.
|
||||
* The bundled Daytona plugin is outside the root workspace; prepare it as CI
|
||||
* does, and rebuild it whenever its sources (or the plugin SDK's) changed since
|
||||
* the last build, so a worktree that moved to a new master tests that master's
|
||||
* plugin rather than an older build.
|
||||
*/
|
||||
export function ensureDaytonaPlugin(repositoryRoot: string, timeline: Timeline) {
|
||||
const root = path.join(repositoryRoot, "packages/plugins/sandbox-providers/daytona");
|
||||
// Inputs outside the plugin directory: the linked plugin SDK and the tsconfig chain both packages extend.
|
||||
const extraInputs = [
|
||||
...["src", "package.json", "tsconfig.json"].map(file => path.join(repositoryRoot, "packages/plugins/sdk", file)),
|
||||
...["tsconfig.json", "tsconfig.base.json"].map(file => path.join(repositoryRoot, file)),
|
||||
];
|
||||
const state = pluginBuildState(root, extraInputs);
|
||||
timeline.add("daytona_plugin_build_state", { reason: state.reason, inputHash: state.inputHash });
|
||||
if (!state.needsBuild) return;
|
||||
run("pnpm", ["install", "--ignore-workspace", "--frozen-lockfile", "--ignore-scripts"], root, "Daytona plugin install");
|
||||
run(process.execPath, [path.join(repositoryRoot, "scripts/link-plugin-dev-sdk.mjs")], repositoryRoot, "Plugin SDK link");
|
||||
run("pnpm", ["--dir", root, "build"], repositoryRoot, "Daytona plugin build");
|
||||
writeFileSync(path.join(root, PLUGIN_BUILD_STAMP), `${state.inputHash}\n`);
|
||||
timeline.add("daytona_plugin_built", { inputHash: state.inputHash });
|
||||
}
|
||||
|
||||
/**
|
||||
* Controller environment for `--env daytona`: extract the image's Linux runnerd
|
||||
* and provider pack (cached per image digest, no Docker), refuse a controller
|
||||
* whose runner source differs from the image, and build the Daytona plugin.
|
||||
*/
|
||||
export async function prepareDaytonaController(input: { image: string; repositoryRoot: string; timeline: Timeline }):
|
||||
Promise<Record<string, string>> {
|
||||
const artifacts = await prepareDaytonaImageArtifacts({ image: input.image, cacheRoot: defaultDaytonaImageCacheRoot(),
|
||||
log: line => console.info(`[daytona-image] ${line}`) });
|
||||
const { localContentId } = await assertControllerMatchesImage(artifacts, input.repositoryRoot);
|
||||
input.timeline.add("daytona_image_matched", { image: input.image, contentId: artifacts.contentId, localContentId, sourceRevision: artifacts.sourceRevision,
|
||||
runnerdSha256: artifacts.runnerdSha256, providerPackDigest: artifacts.providerPackDigest, platformManifestDigest: artifacts.platformManifestDigest });
|
||||
ensureDaytonaPlugin(input.repositoryRoot, input.timeline);
|
||||
return { PAPERCLIP_RUNNER_REMOTE_BINARY_PATH: artifacts.runnerdPath, PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH: artifacts.providerPackPath };
|
||||
}
|
||||
@@ -207,6 +207,7 @@ async function main(): Promise<number> {
|
||||
const api = new ChaosApi(supervisor.baseUrl);
|
||||
const lab = await seedChaosLab({ api, env: plan.options.env, daytonaImage: plan.options.daytonaImage, daytonaCpu: plan.options.daytonaCpu,
|
||||
daytonaReuseLease: plan.options.daytonaReuseLease,
|
||||
daytonaWarm: plan.options.daytonaWarm, daytonaInherit: plan.options.daytonaInherit,
|
||||
adapters: [...new Set(plan.tasks.map(task => task.adapter))],
|
||||
concurrency: plan.options.concurrency, budgetCents: plan.options.budgetCents, masks: plan.options.masks,
|
||||
fixtureRoot: path.join(supervisor.paths.temporaryRoot, "fixtures"), waveId: plan.waveId, timeline,
|
||||
|
||||
@@ -2,7 +2,7 @@ import { execFileSync } from "node:child_process";
|
||||
import { chmod, mkdir, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { RunnerApi } from "../runner-e2e/api.js";
|
||||
import { runnerEnvironments, runnerProfiles } from "../runner-e2e/catalog.js";
|
||||
import { daytonaWarmEnvironment, runnerEnvironments, runnerProfiles } from "../runner-e2e/catalog.js";
|
||||
import { installFixtureDaytonaProvider, waitForFixtureDaytonaProvider } from "../runner-e2e/live-fixtures.js";
|
||||
import type { MatrixExecution, SecretReference, SecretReferenceMap } from "../runner-e2e/types.js";
|
||||
import type { ChaosApi } from "./api.js";
|
||||
@@ -60,13 +60,20 @@ export function chaosDaytonaShape(cpu: number) {
|
||||
}
|
||||
|
||||
async function createDaytonaEnvironment(input: { api: ChaosApi; companyId: string; image: string; cpu: number; reuseLease: boolean; waveId: string;
|
||||
secretRefs: SecretReferenceMap; timeline: Timeline }): Promise<string> {
|
||||
const fixture = runnerEnvironments.find(candidate => candidate.id === "daytona");
|
||||
secretRefs: SecretReferenceMap; timeline: Timeline; warm?: boolean; inherit?: boolean }): Promise<string> {
|
||||
// The warm fixture keeps a reusable sandbox and a warm runner between turns.
|
||||
const fixture = input.warm ? daytonaWarmEnvironment : runnerEnvironments.find(candidate => candidate.id === "daytona");
|
||||
if (!fixture) throw new Error("runner-e2e catalog has no daytona environment fixture");
|
||||
const view = runnerApiView(input.api);
|
||||
await waitForFixtureDaytonaProvider(view, { companyId: input.companyId, deadlineAt: Date.now() + 120_000 });
|
||||
const payload = fixture.buildEnvironment({ secretRefs: input.secretRefs, daytonaImage: input.image, executionId: input.waveId });
|
||||
const shape = chaosDaytonaShape(input.cpu);
|
||||
// A user's default environment: the agent decides the lifecycle, and nothing pins lease reuse.
|
||||
if (input.inherit) {
|
||||
const config = payload.config as Record<string, unknown>;
|
||||
delete config.runnerLifecycleMode;
|
||||
delete config.reuseLease;
|
||||
}
|
||||
const environment = await input.api.post<{ id: string; driver: string }>(`/api/companies/${input.companyId}/environments`,
|
||||
{ ...payload, config: { ...(payload.config as Record<string, unknown>), ...shape, ...(input.reuseLease ? { reuseLease: true } : {}) } });
|
||||
input.timeline.add("seed_environment", { environmentId: environment.id, driver: environment.driver, image: input.image, ...shape,
|
||||
@@ -120,6 +127,12 @@ export async function seedChaosLab(input: {
|
||||
timeline: Timeline;
|
||||
/** Merged into each agent's adapterConfig (for example a runner lifecycle mode). */
|
||||
adapterConfigOverrides?: Record<string, unknown>;
|
||||
/** Use the reusable warm Daytona environment fixture (reuseLease, warm runner). */
|
||||
daytonaWarm?: boolean;
|
||||
/** Use a Daytona environment that inherits the lifecycle and leaves lease reuse unset, as a user's default does. */
|
||||
daytonaInherit?: boolean;
|
||||
/** A real repository to use as the project's primary workspace instead of the tiny fixture. */
|
||||
repositoryPath?: string;
|
||||
/** Called as soon as the wave owns external resources that need cleanup. */
|
||||
onEnvironment?: (lab: Pick<ChaosLab, "companyId" | "environmentId" | "environmentKind">) => void;
|
||||
}): Promise<ChaosLab> {
|
||||
@@ -130,7 +143,8 @@ export async function seedChaosLab(input: {
|
||||
await installFixtureDaytonaProvider(runnerApiView(input.api), { environment: { id: "daytona" } } as MatrixExecution);
|
||||
timeline.add("seed_daytona_plugin", {});
|
||||
}
|
||||
const fixtureRepositoryPath = await createFixtureRepository(path.join(input.fixtureRoot, "chaos-sandbox"));
|
||||
const fixtureRepositoryPath = input.repositoryPath
|
||||
?? await createFixtureRepository(path.join(input.fixtureRoot, "chaos-sandbox"));
|
||||
const company = await api.post<{ id: string }>("/api/companies", {
|
||||
name: "Chaos Lab",
|
||||
description: `Restart chaos harness wave ${input.waveId}`,
|
||||
@@ -161,7 +175,7 @@ export async function seedChaosLab(input: {
|
||||
let environmentId: string;
|
||||
if (input.env === "daytona") {
|
||||
environmentId = await createDaytonaEnvironment({ api, companyId: company.id, image: input.daytonaImage!, cpu: input.daytonaCpu ?? 4,
|
||||
reuseLease: input.daytonaReuseLease === true, waveId: input.waveId, secretRefs, timeline });
|
||||
reuseLease: input.daytonaReuseLease === true, waveId: input.waveId, secretRefs, timeline, warm: input.daytonaWarm, inherit: input.daytonaInherit });
|
||||
input.onEnvironment?.({ companyId: company.id, environmentId, environmentKind: "daytona" });
|
||||
if (input.adapters.includes("codex")) await qualifyNativeCodex({ api, companyId: company.id, environmentId, timeline });
|
||||
} else {
|
||||
|
||||
@@ -16,7 +16,7 @@ export interface BaselineRun {
|
||||
adapter: string;
|
||||
environment: string;
|
||||
/** Driver intent: a new task, a follow-up comment on the same task, or a run crossed by a restart. */
|
||||
phase: "cold" | "warm" | "restart" | "midturn" | "verify";
|
||||
phase: "cold" | "warm" | "restart" | "midturn" | "verify" | "postfault";
|
||||
events: BaselineEvent[];
|
||||
}
|
||||
|
||||
@@ -149,8 +149,8 @@ export interface BaselineRow { adapter: string; environment: string; warmth: "co
|
||||
export function baselineRows(runs: readonly BaselineRun[]): BaselineRow[] {
|
||||
const groups = new Map<string, { adapter: string; environment: string; warmth: "cold" | "warm"; metrics: RunMetrics[] }>();
|
||||
for (const run of runs) {
|
||||
// Restart samples and new-agent verification samples are reported on their own.
|
||||
if (run.phase === "restart" || run.phase === "verify") continue;
|
||||
// Restart, post-restart and new-agent verification samples are reported on their own.
|
||||
if (run.phase === "restart" || run.phase === "verify" || run.phase === "postfault") continue;
|
||||
const warmth = runWarmth(run);
|
||||
const key = `${run.adapter}|${run.environment}|${warmth}`;
|
||||
const group = groups.get(key) ?? { adapter: run.adapter, environment: run.environment, warmth, metrics: [] };
|
||||
|
||||
@@ -20,11 +20,13 @@ import path from "node:path";
|
||||
import { parseArgs } from "node:util";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { ChaosApi } from "./api.js";
|
||||
import { CREDENTIAL_FOR_ADAPTER } from "./config.js";
|
||||
import { CREDENTIAL_FOR_ADAPTER, DAYTONA_VCPU_BUDGET } from "./config.js";
|
||||
import { ACTIVE_RUN_STATUSES } from "./grader.js";
|
||||
import { chaosLifecycleOverrides, seedChaosLab, type ChaosLab } from "./seed.js";
|
||||
import { ChaosSupervisor } from "./supervisor.js";
|
||||
import { createDb } from "../../packages/db/src/index.js";
|
||||
import { prepareDaytonaController } from "./daytona-controller.js";
|
||||
import { DaytonaWaveSweeper } from "./daytona-sweep.js";
|
||||
import { Timeline } from "./timeline.js";
|
||||
import type { AdapterId, FaultKind } from "./types.js";
|
||||
import {
|
||||
@@ -88,9 +90,13 @@ async function steerQueuedComment(api: ChaosApi, issueId: string, commentId: str
|
||||
return { acknowledgedAt: null, disposition };
|
||||
}
|
||||
|
||||
/** Set by --workspace-repo: tasks then run in the project's isolated workspace of that repository. */
|
||||
let projectScopedTasks = false;
|
||||
|
||||
async function createTask(api: ChaosApi, lab: ChaosLab, adapter: AdapterId, title: string, description: string) {
|
||||
const issue = await api.post<Json>(`/api/companies/${lab.companyId}/issues`, {
|
||||
title, description, status: "todo", priority: "medium", assigneeAgentId: lab.agents[adapter].id,
|
||||
...(projectScopedTasks ? { projectId: lab.projectId } : {}),
|
||||
});
|
||||
return String(issue.id);
|
||||
}
|
||||
@@ -200,6 +206,11 @@ function report(runs: BaselineRun[], faults: BaselineFault[]): string {
|
||||
lines.push("", "New agent behind harness verification (TTFT counts from run creation; the verification wait precedes it):", "", lines[0]!, lines[1]!);
|
||||
for (const row of baselineRows(verifyRuns)) lines.push(rowLine(row));
|
||||
}
|
||||
const postfault = runs.filter(run => run.phase === "postfault").map(run => ({ ...run, phase: "cold" as const }));
|
||||
if (postfault.length) {
|
||||
lines.push("", "First follow-up after an idle control-plane restart (K1):", "", lines[0]!, lines[1]!);
|
||||
for (const row of baselineRows(postfault)) lines.push(rowLine(row));
|
||||
}
|
||||
|
||||
lines.push("", "| Pre-turn step | n | p50 | p95 | Adapters |", "|---|---|---|---|---|");
|
||||
for (const step of rankPreTurnSteps(runs).slice(0, 25)) lines.push(`| \`${step.step}\` | ${step.n} | ${ms(step.p50)} | ${ms(step.p95)} | ${step.adapters.join(", ")} |`);
|
||||
@@ -236,6 +247,17 @@ async function main() {
|
||||
"midturn-fault": { type: "string" },
|
||||
// verify phase: fault the control plane while a newly hired agent runs harness verification.
|
||||
"verify-fault": { type: "string" },
|
||||
// A real repository (for example a fresh clone of paperclip) as the project's
|
||||
// primary workspace; every task then runs in its isolated git worktree.
|
||||
"workspace-repo": { type: "string" },
|
||||
// local (default) or daytona; daytona needs --daytona-image (the signed Runner image).
|
||||
env: { type: "string", default: "local" },
|
||||
"daytona-image": { type: "string" },
|
||||
"daytona-cpu": { type: "string", default: "4" },
|
||||
// Use the reusable warm Daytona environment (reuseLease, warm runner) instead of the per-turn one.
|
||||
"daytona-warm": { type: "boolean", default: false },
|
||||
// A Daytona environment as a user creates it by default: lifecycle inherited, lease reuse unset.
|
||||
"daytona-inherit": { type: "boolean", default: false },
|
||||
"idle-ms": { type: "string", default: "600000" },
|
||||
} });
|
||||
if (values.report) {
|
||||
@@ -247,7 +269,27 @@ async function main() {
|
||||
const requested = values.adapters!.split(",").map(value => value.trim()) as AdapterId[];
|
||||
const samples = Math.max(1, Number(values.samples));
|
||||
const restarts = Math.max(0, Number(values.restarts));
|
||||
const missing = requested.map(adapter => CREDENTIAL_FOR_ADAPTER[adapter]).filter(key => !process.env[key]);
|
||||
const env = values.env === "daytona" ? "daytona" as const : "local" as const;
|
||||
if (env === "daytona" && !values["daytona-image"]) throw new Error("--env daytona needs --daytona-image");
|
||||
if (env === "daytona") {
|
||||
// A warm sandbox (the warm fixture, or the inheriting default) outlives its task for
|
||||
// the idle window and no other task reuses it, so count one per sampled task.
|
||||
const budget = Number(process.env.RESTART_CHAOS_DAYTONA_VCPU_BUDGET ?? DAYTONA_VCPU_BUDGET);
|
||||
const perTaskPhases = ["coldwarm", "midturn", "verify"].filter(phase => phases.has(phase)).length
|
||||
+ (phases.has("restart") ? Math.max(1, Number(values.restarts)) : 0);
|
||||
const cpu = Number(values["daytona-cpu"]);
|
||||
// The default fixture releases each sandbox after its turn; only a warm one is retained.
|
||||
const retainsSandboxes = values["daytona-warm"] === true || values["daytona-inherit"] === true;
|
||||
const holders = retainsSandboxes
|
||||
? requested.length * Math.max(1, Number(values.samples)) * Math.max(1, perTaskPhases)
|
||||
: requested.length;
|
||||
if (holders * cpu > budget) {
|
||||
throw new Error(`${holders} ${retainsSandboxes ? "sampled tasks (warm sandboxes outlive their task)" : "concurrent tasks"} × --daytona-cpu ${cpu} vCPU can be held at once, `
|
||||
+ `exceeding the Daytona vCPU budget of ${budget}. Lower --samples, --adapters or --daytona-cpu (RESTART_CHAOS_DAYTONA_VCPU_BUDGET sets the budget).`);
|
||||
}
|
||||
}
|
||||
const missing = [...requested.map(adapter => CREDENTIAL_FOR_ADAPTER[adapter]), ...(env === "daytona" ? ["DAYTONA_API_KEY"] : [])]
|
||||
.filter(key => !process.env[key]);
|
||||
if (missing.length) throw new Error(`Missing ${missing.join(", ")}; run: set -a; source ~/.secrets; set +a`);
|
||||
const midturnMode = values["midturn-mode"];
|
||||
if (midturnMode !== "queue" && midturnMode !== "steer") throw new Error("--midturn-mode must be queue or steer");
|
||||
@@ -276,10 +318,34 @@ async function main() {
|
||||
const agentLifecycles: Record<string, string> = {};
|
||||
const midturn: Json[] = [];
|
||||
const timedOutIssues = new Set<string>();
|
||||
let sweeper: DaytonaWaveSweeper | null = null;
|
||||
const workspaceOperations: Array<{ runId: string; adapter: string; phase: string; operation: string; status: string;
|
||||
durationMs: number | null; command: string | null }> = [];
|
||||
const followUpTasks = new Map<AdapterId, { issueId: string; nonce: string; runs: number }>();
|
||||
// One shutdown path for normal completion and for SIGINT/SIGTERM: the server
|
||||
// runs in a detached process group and the embedded Postgres has no exit hook.
|
||||
let stopping: Promise<void> | null = null;
|
||||
const shutdown = () => (stopping ??= supervisor.stop().catch(error => {
|
||||
// Daytona resources are released while the server still runs, before it stops;
|
||||
// the same path serves normal completion and SIGINT/SIGTERM.
|
||||
const releaseDaytona = async () => {
|
||||
if (!sweeper) return;
|
||||
const owned = sweeper as DaytonaWaveSweeper;
|
||||
sweeper = null;
|
||||
try {
|
||||
const cleanup = await owned.cleanUp();
|
||||
timeline.add("daytona_cleanup", cleanup);
|
||||
await writeFile(path.join(outDir, "sandbox-cleanup.json"), `${JSON.stringify(cleanup, null, 2)}\n`, { mode: 0o600 });
|
||||
if (cleanup.remainingSandboxes.length || cleanup.errors.length || cleanup.environmentDeleteBlocked || !cleanup.environmentDeleted) {
|
||||
process.stderr.write(`Daytona cleanup incomplete: remaining ${cleanup.remainingSandboxes.join(", ") || "none"}; `
|
||||
+ `environment deleted ${cleanup.environmentDeleted}${cleanup.environmentDeleteBlocked ? ` (blocked: ${String(cleanup.environmentDeleteBlocked)})` : ""}; ${cleanup.errors.join("; ")}\n`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
} catch (error) {
|
||||
timeline.add("daytona_cleanup_failed", { error: String(error) });
|
||||
process.exitCode = 1;
|
||||
}
|
||||
};
|
||||
const shutdown = () => (stopping ??= releaseDaytona().then(() => supervisor.stop()).catch(error => {
|
||||
process.stderr.write(`supervisor stop: ${String(error)}\n`);
|
||||
}));
|
||||
const onSignal = (signal: NodeJS.Signals) => {
|
||||
@@ -289,10 +355,18 @@ async function main() {
|
||||
process.once("SIGINT", onSignal);
|
||||
process.once("SIGTERM", onSignal);
|
||||
try {
|
||||
await supervisor.start();
|
||||
await supervisor.start(env === "daytona"
|
||||
? await prepareDaytonaController({ image: values["daytona-image"]!, repositoryRoot, timeline }) : {});
|
||||
const api = new ChaosApi(supervisor.baseUrl);
|
||||
const lab = await seedChaosLab({ api, env: "local", adapters: requested, concurrency: requested.length, budgetCents: 3000,
|
||||
masks: ["rc06"], fixtureRoot: supervisor.paths.temporaryRoot, waveId, timeline, adapterConfigOverrides });
|
||||
const repositoryPath = values["workspace-repo"] ? path.resolve(values["workspace-repo"]) : undefined;
|
||||
projectScopedTasks = Boolean(repositoryPath);
|
||||
const lab = await seedChaosLab({ api, env, daytonaImage: values["daytona-image"], daytonaCpu: Number(values["daytona-cpu"]),
|
||||
daytonaWarm: values["daytona-warm"], daytonaInherit: values["daytona-inherit"],
|
||||
adapters: requested, concurrency: requested.length, budgetCents: 3000,
|
||||
masks: ["rc06"], fixtureRoot: supervisor.paths.temporaryRoot, waveId, timeline, adapterConfigOverrides, repositoryPath,
|
||||
onEnvironment: owned => {
|
||||
sweeper = new DaytonaWaveSweeper({ apiKey: process.env.DAYTONA_API_KEY!, companyId: owned.companyId, environmentId: owned.environmentId, api });
|
||||
} });
|
||||
const adapters = await readyAdapters(api, lab, requested, timeline);
|
||||
if (!adapters.length) throw new Error("No agent passed lifecycle verification");
|
||||
// Record what each agent actually stores: "unset" means the server resolves
|
||||
@@ -328,6 +402,7 @@ async function main() {
|
||||
continue;
|
||||
}
|
||||
timeline.add("baseline_warm_done", { adapter, issueId, runs: warm.runs.length - cold.runs.length });
|
||||
followUpTasks.set(adapter, { issueId, nonce, runs: warm.runs.length });
|
||||
}
|
||||
}));
|
||||
|
||||
@@ -383,6 +458,29 @@ async function main() {
|
||||
}
|
||||
}
|
||||
|
||||
// First turn after an idle control-plane restart: K1 while nothing runs,
|
||||
// then one more follow-up on each task the cold/warm phase left at rest.
|
||||
// The fault must be idle: no timed-out sample may still be running.
|
||||
const allAtRest = timedOutIssues.size === 0 && (await Promise.all(tasks.map(async task =>
|
||||
(await runsFor(api, task.issueId)).every(run => !ACTIVE_RUN_STATUSES.has(String(run.status)))))).every(Boolean);
|
||||
if (phases.has("postfault") && followUpTasks.size && !allAtRest) {
|
||||
timeline.add("baseline_postfault_skipped", { reason: "work_not_at_rest", timedOut: timedOutIssues.size });
|
||||
}
|
||||
if (phases.has("postfault") && followUpTasks.size && allAtRest) {
|
||||
const receipt = await supervisor.fault("K1", 0, "ttft-baseline-postfault");
|
||||
faults.push({ kind: "K1", readyAt: receipt.readyAt ?? Date.now(), runIds: [] });
|
||||
await Promise.all([...followUpTasks.entries()].map(async ([adapter, entry]) => {
|
||||
await api.post(`/api/issues/${entry.issueId}/comments`, { body: warmFollowUp(`${entry.nonce}-postfault`), reopen: true });
|
||||
const after = await waitForRest(api, supervisor, entry.issueId, entry.runs + 1);
|
||||
for (const run of after.runs) if (!runPhase.has(runIdOf(run))) runPhase.set(runIdOf(run), "postfault");
|
||||
if (after.timedOut) {
|
||||
timedOutIssues.add(entry.issueId);
|
||||
timeline.add("baseline_sample_timeout", { adapter, issueId: entry.issueId, phase: "postfault" });
|
||||
}
|
||||
else timeline.add("baseline_postfault_done", { adapter, issueId: entry.issueId });
|
||||
}));
|
||||
}
|
||||
|
||||
// Mid-turn follow-up: a comment posted while the agent is inside a long
|
||||
// tool call. Records when the comment landed and every run of the task,
|
||||
// so the wait splits into "rest of the turn" and "hand-off".
|
||||
@@ -472,15 +570,34 @@ async function main() {
|
||||
if (timedOutIssues.has(task.issueId)) continue;
|
||||
for (const run of await runsFor(api, task.issueId)) {
|
||||
const runId = runIdOf(run);
|
||||
runs.push({ runId, adapter: task.adapter, environment: "local",
|
||||
runs.push({ runId, adapter: task.adapter, environment: env,
|
||||
phase: runPhase.get(runId) ?? task.phase, events: await allEvents(api, runId) });
|
||||
// Workspace commands (worktree, provision, sync) the run waited on, with their durations.
|
||||
const operations = await api.get<Json[]>(`/api/heartbeat-runs/${runId}/workspace-operations`, 10_000).catch((error: unknown) => {
|
||||
// A failed read is recorded as such, never as "no workspace operations".
|
||||
workspaceOperations.push({ runId, adapter: task.adapter, phase: runPhase.get(runId) ?? task.phase,
|
||||
operation: "read_failed", status: error instanceof Error ? error.message.slice(0, 160) : String(error), durationMs: null, command: null });
|
||||
return [] as Json[];
|
||||
});
|
||||
for (const operation of operations) workspaceOperations.push({ runId, adapter: task.adapter,
|
||||
phase: runPhase.get(runId) ?? task.phase, operation: String(operation.phase), status: String(operation.status),
|
||||
durationMs: operation.startedAt && operation.finishedAt
|
||||
? Date.parse(String(operation.finishedAt)) - Date.parse(String(operation.startedAt)) : null,
|
||||
command: typeof operation.command === "string" ? operation.command.slice(0, 120) : null });
|
||||
}
|
||||
}
|
||||
const data = { waveId, lifecycle: lifecycle ?? "catalog", agentLifecycles, adapters, notReady: requested.filter(adapter => !adapters.includes(adapter)),
|
||||
const data = { workspaceOperations, waveId, lifecycle: lifecycle ?? "catalog", agentLifecycles, adapters, notReady: requested.filter(adapter => !adapters.includes(adapter)),
|
||||
timedOutTasks: timedOutIssues.size, samples, restarts, midturn, gitSha: process.env.GIT_SHA ?? null, runs, faults };
|
||||
await writeFile(path.join(outDir, "baseline-data.json"), `${JSON.stringify(data)}\n`, { mode: 0o600 });
|
||||
await writeFile(path.join(outDir, "baseline-report.md"), report(runs, faults), { mode: 0o600 });
|
||||
process.stdout.write(report(runs, faults));
|
||||
if (workspaceOperations.length) {
|
||||
const table = ["", "| Run phase | Adapter | Workspace operation | Status | Duration | Command |", "|---|---|---|---|---|---|",
|
||||
...workspaceOperations.slice().sort((a, b) => (b.durationMs ?? 0) - (a.durationMs ?? 0))
|
||||
.map(op => `| ${op.phase} | ${op.adapter} | ${op.operation} | ${op.status} | ${op.durationMs === null ? "—" : `${(op.durationMs / 1000).toFixed(2)} s`} | ${op.command ? `\`${op.command}\`` : "—"} |`)].join("\n");
|
||||
process.stdout.write(`${table}\n`);
|
||||
await writeFile(path.join(outDir, "workspace-operations.md"), `${table}\n`, { mode: 0o600 });
|
||||
}
|
||||
if (timedOutIssues.size) process.stdout.write(`\n${timedOutIssues.size} sample(s) timed out and were left out.\n`);
|
||||
// Every answered mid-turn comment must be answered exactly once.
|
||||
const wrongReplies = midturn.filter(sample => sample.replyCount !== 1);
|
||||
@@ -491,7 +608,8 @@ async function main() {
|
||||
if (wrongReplies.length) process.exitCode = 1;
|
||||
process.stdout.write(`\nWrote ${outDir}\n`);
|
||||
} finally {
|
||||
// Always stop the wrapper-owned embedded Postgres (SysV shared memory is scarce).
|
||||
// Every Daytona sandbox this run created is archived or destroyed before the server
|
||||
// stops, then the wrapper-owned embedded Postgres stops (SysV shared memory is scarce).
|
||||
await shutdown();
|
||||
process.off("SIGINT", onSignal);
|
||||
process.off("SIGTERM", onSignal);
|
||||
|
||||
@@ -36,6 +36,10 @@ export interface WaveOptions {
|
||||
daytonaCpu?: number;
|
||||
/** `--daytona-reuse-lease`: the chaos environment keeps its sandboxes between turns (a per-turn run stops and keeps its sandbox). */
|
||||
daytonaReuseLease?: boolean;
|
||||
/** `--daytona-warm`: the reusable warm Daytona environment fixture. */
|
||||
daytonaWarm?: boolean;
|
||||
/** `--daytona-inherit`: a Daytona environment that inherits the lifecycle and leaves lease reuse unset. */
|
||||
daytonaInherit?: boolean;
|
||||
/** `--k4-hold-max-seconds`: upper end of the seeded K4 hold (default 300 s, cap 600 s). */
|
||||
k4HoldMaxMs?: number;
|
||||
/**
|
||||
|
||||
Reference in new issue
Block a user