diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 1a0c42eba5c..c638a17a2a0 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -24,6 +24,16 @@ node scripts/typecheck-test-projects.mjs tests/restart-chaos # typecheck Options: `--triggers time,comment,retry,queued,tool` (fault timing, below), `--workloads A1,B1,B5,C1`, `--budget-cents` (company hard cap, default 3000), `--poll-ms` (2000–5000), `--keep` (keep the temporary instance). + +`--rc06-masked` keeps the company budget amount but disables its hard stop on +the isolated Chaos Lab company, so one stuck accounting row (RC-06) cannot +wedge every later task. Masked tasks are marked `rc06-masked` in the ledger and +summary, and each wave still records its spend (`wave_spend` in the timeline). +A replay inherits the masks saved in the source wave's `plan.json`; +`--replay / --rc06-masked` adds the mask to a replay of an unmasked wave. +Each fault receipt records the host 1-minute load average (`loadAvg1m`); treat +timeout failures above about 16 as load-suspect. + ## Daytona (`--env daytona`) The control plane stays local; agent runs execute in Daytona sandboxes from an diff --git a/tests/restart-chaos/config.ts b/tests/restart-chaos/config.ts index bf5b45bd43c..29d1cd33ca2 100644 --- a/tests/restart-chaos/config.ts +++ b/tests/restart-chaos/config.ts @@ -30,7 +30,7 @@ export const DAYTONA_VCPU_BUDGET = 40; export type ParsedCommand = | { mode: "wave"; options: WaveOptions; keep: boolean } - | { mode: "replay"; ref: ReplayRef; keep: boolean; overrides: Partial> }; + | { mode: "replay"; ref: ReplayRef; keep: boolean; overrides: Partial> }; export interface ReplayRef { waveId: string; taskId: string } @@ -84,6 +84,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC "daytona-image": { type: "string" }, "daytona-cpu": { type: "string" }, keep: { type: "boolean" }, + "rc06-masked": { type: "boolean" }, }, }); const keep = values.keep === true; @@ -94,7 +95,9 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC if (values[key] !== undefined) throw new Error(`--replay reproduces the saved plan; do not combine it with --${key}`); } return { mode: "replay", ref: parseReplayRef(values.replay), keep, - overrides: { ...(values["poll-ms"] ? { pollMs } : {}), ...(values["budget-cents"] ? { budgetCents } : {}) } }; + overrides: { ...(values["poll-ms"] ? { pollMs } : {}), ...(values["budget-cents"] ? { budgetCents } : {}), + // The saved plan already carries its masks; this adds the RC-06 mask to a replay of an unmasked wave. + ...(values["rc06-masked"] ? { masks: ["rc06"] } : {}) } }; } const env = (values.env ?? "local") as EnvironmentKind; if (!ENVIRONMENT_KINDS.includes(env)) throw new Error(`Unknown env "${env}"; allowed: ${ENVIRONMENT_KINDS.join(", ")}`); @@ -118,6 +121,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC budgetCents, pollMs, ...(env === "daytona" ? { daytonaImage, daytonaCpu: integer(values["daytona-cpu"], "daytona-cpu", DAYTONA_DEFAULT_CPU, 1, 4) } : {}), + ...(values["rc06-masked"] ? { masks: ["rc06"] } : {}), }; if (env !== "daytona" && values["daytona-cpu"] !== undefined) throw new Error("--daytona-cpu applies only to --env daytona"); if (options.daytonaCpu !== undefined) { diff --git a/tests/restart-chaos/ledger.ts b/tests/restart-chaos/ledger.ts index b08c053e2ae..57a5c7ea87f 100644 --- a/tests/restart-chaos/ledger.ts +++ b/tests/restart-chaos/ledger.ts @@ -30,6 +30,8 @@ export interface LedgerLine { evidence: string | null; replay: string; recordedAt: string; + /** Known signatures masked for this wave, e.g. ["rc06"]. */ + masks?: string[]; } export function ledgerLine(plan: WavePlan, record: TaskRecord, grade: TaskGrade, gitSha: string, evidence: string | null, @@ -46,6 +48,7 @@ export function ledgerLine(plan: WavePlan, record: TaskRecord, grade: TaskGrade, ...(loads.length ? { maxLoadAvg1m: Math.max(...loads) } : {}), signature: grade.signature, retried: Boolean(record.retry), evidence, replay: replayCommand(plan.waveId, record.plan.taskId), recordedAt: new Date().toISOString(), + ...(plan.options.masks?.length ? { masks: [...plan.options.masks] } : {}), }; } @@ -64,7 +67,7 @@ export function renderSummary(lines: readonly LedgerLine[], latestWave?: string) const cells = new Map(); const signatures = new Map(); for (const line of lines) { - const key = `${line.env} · ${line.adapter} · ${line.workload} · ${line.fault}${line.trigger ? ` ^${line.trigger}` : ""}`; + const key = `${line.env} · ${line.adapter} · ${line.workload} · ${line.fault}${line.trigger ? ` ^${line.trigger}` : ""}${line.masks?.length ? ` · ${line.masks.map(mask => `${mask}-masked`).join(",")}` : ""}`; const cell = cells.get(key) ?? { pass: 0, total: 0, crossed: 0 }; cell.total += 1; if (line.pass) cell.pass += 1; if (line.crossed) cell.crossed += 1; cells.set(key, cell); @@ -78,7 +81,7 @@ export function renderSummary(lines: readonly LedgerLine[], latestWave?: string) if (latestWave) { out.push(`## Latest wave ${latestWave}`, "", "| Task | Cell | Crossed | R1 | R2 | R3 | R4 | R5 | R6 | T | Result |", "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"); for (const line of lines.filter(candidate => candidate.waveId === latestWave)) { - out.push(`| ${line.taskId} ${line.issue ?? ""} | ${line.adapter} ${line.workload} ${line.fault}${line.trigger ? ` ^${line.trigger}${line.triggerMissed?.length ? " (missed)" : ""}` : ""} | ${line.crossed ? "yes" : "no"} | ${line.rules.R1} | ${line.rules.R2} | ${line.rules.R3} | ${line.rules.R4} | ${line.rules.R5} | ${line.rules.R6} | ${line.T} | ${line.pass ? "PASS" : `FAIL ${line.signature}`} |`); + out.push(`| ${line.taskId} ${line.issue ?? ""} | ${line.adapter} ${line.workload} ${line.fault}${line.trigger ? ` ^${line.trigger}${line.triggerMissed?.length ? " (missed)" : ""}` : ""}${line.masks?.length ? ` (${line.masks.map(mask => `${mask}-masked`).join(", ")})` : ""} | ${line.crossed ? "yes" : "no"} | ${line.rules.R1} | ${line.rules.R2} | ${line.rules.R3} | ${line.rules.R4} | ${line.rules.R5} | ${line.rules.R6} | ${line.T} | ${line.pass ? "PASS" : `FAIL ${line.signature}`} |`); } out.push(""); } diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index 60cea2129e7..e3a97c19d5e 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -125,6 +125,18 @@ async function main(): Promise { harnessError ??= error; interrupted = true; }); + // Spend is read once per wave: after the tasks, or during cleanup when the wave stopped early. + let spendTarget: { api: ChaosApi; companyId: string } | null = null; + let spendRecorded = false; + const recordWaveSpend = async () => { + if (!spendTarget || spendRecorded) return; + spendRecorded = true; + // Each wave owns its company, so all-time spend is exactly this wave, even across a month boundary. + const spend = await spendTarget.api.get>(`/api/companies/${spendTarget.companyId}/costs/summary?period=all`) + .catch(error => ({ error: String(error) })); + timeline.add("wave_spend", { spend }); + console.info(`Wave spend: ${JSON.stringify(spend)}`); + }; let faultFailures = 0; let faultsQueued = 0; try { @@ -132,11 +144,12 @@ async function main(): Promise { const api = new ChaosApi(supervisor.baseUrl); const lab = await seedChaosLab({ api, env: plan.options.env, daytonaImage: plan.options.daytonaImage, daytonaCpu: plan.options.daytonaCpu, adapters: [...new Set(plan.tasks.map(task => task.adapter))], - concurrency: plan.options.concurrency, budgetCents: plan.options.budgetCents, + concurrency: plan.options.concurrency, budgetCents: plan.options.budgetCents, masks: plan.options.masks, fixtureRoot: path.join(supervisor.paths.temporaryRoot, "fixtures"), waveId: plan.waveId, timeline, onEnvironment: owned => { sweeper = new DaytonaWaveSweeper({ apiKey: process.env.DAYTONA_API_KEY!, companyId: owned.companyId, environmentId: owned.environmentId, api }); } }); + spendTarget = { api, companyId: lab.companyId }; // The instance root holds the secrets master key and the database: oracle code may not read it. const protectedRoots = [os.homedir(), supervisor.paths.temporaryRoot, resultsRoot]; const observers = plan.tasks.map(task => new TaskObserver(task, api, lab, timeline, supervisor, protectedRoots)); @@ -193,6 +206,7 @@ async function main(): Promise { // R6: let finalizers settle, then sweep for anything still pending. const quietFor = Date.now() - lastDoneAt; if (quietFor < QUIET_BEFORE_SWEEP_MS) await delay(QUIET_BEFORE_SWEEP_MS - quietFor); + await recordWaveSpend(); timeline.add("wave_sweep", {}); if (sweeper) { sandboxSweep = await (sweeper as DaytonaWaveSweeper).sweep(); @@ -238,6 +252,8 @@ async function main(): Promise { harnessError = error; timeline.add("harness_error", { error: error instanceof Error ? error.stack ?? error.message : String(error) }); } finally { + // Best effort for an interrupted or failed wave; the server may already be down. + if (supervisor.serverReady) await recordWaveSpend().catch(() => undefined); // A failure here (for example writing the report) must not skip stopping the controller and Postgres. try { if (sweeper) try { const owned = sweeper as DaytonaWaveSweeper; diff --git a/tests/restart-chaos/scheduler.test.ts b/tests/restart-chaos/scheduler.test.ts index 5445c805e79..8f883a4c150 100644 --- a/tests/restart-chaos/scheduler.test.ts +++ b/tests/restart-chaos/scheduler.test.ts @@ -67,6 +67,20 @@ describe("replay", () => { expect(again.waveId).toBe(`${plan.waveId}-replay-t03-1700000000001`); }); + it("keeps the source wave's masks on replay and lets --rc06-masked add the RC-06 mask", () => { + const masked = planWave("w-20261106T101112-s42", { ...plan.options, masks: ["rc06"] }); + expect(replayWave(masked, { waveId: masked.waveId, taskId: "t01" }, 1).options.masks).toEqual(["rc06"]); + expect(replayWave(plan, { waveId: plan.waveId, taskId: "t01" }, 1).options.masks).toBeUndefined(); + const command = parseCommand(["--replay", `${plan.waveId}/t01`, "--rc06-masked"]); + if (command.mode !== "replay") throw new Error("expected replay"); + expect(command.overrides.masks).toEqual(["rc06"]); + expect(replayWave(plan, { waveId: plan.waveId, taskId: "t01" }, 1, command.overrides).options.masks).toEqual(["rc06"]); + expect(replayWave(masked, { waveId: masked.waveId, taskId: "t01" }, 1, command.overrides).options.masks).toEqual(["rc06"]); + const wave = parseCommand(["--rc06-masked", "--seed", "3"]); + if (wave.mode !== "wave") throw new Error("expected wave"); + expect(wave.options.masks).toEqual(["rc06"]); + }); + it("replays the recorded fault sequence that crossed the task, other tasks' faults included", () => { const createdAt = 1_000_000; const faults = [ diff --git a/tests/restart-chaos/scheduler.ts b/tests/restart-chaos/scheduler.ts index 6d2fff21f2d..b9f307fa349 100644 --- a/tests/restart-chaos/scheduler.ts +++ b/tests/restart-chaos/scheduler.ts @@ -80,6 +80,12 @@ export function planTasks(options: Pick> = {}, observed: ScheduledFault[] = []): WavePlan { + overrides: Partial> = {}, observed: ScheduledFault[] = []): WavePlan { if (source.schema !== "paperclip.restart-chaos.wave-plan/v1") throw new Error("Unsupported wave plan schema"); if (source.waveId !== ref.waveId) throw new Error(`Plan is for ${source.waveId}, not ${ref.waveId}`); const task = source.tasks.find(candidate => candidate.taskId === ref.taskId); @@ -125,7 +131,7 @@ export function replayWave(source: WavePlan, ref: ReplayRef, now = Date.now(), waveId: `${base}-replay-${task.taskId}-${now}`, seed: source.seed, createdAt: new Date(now).toISOString(), - options: { ...source.options, ...overrides, tasks: 1, concurrency: 1 }, + options: { ...source.options, ...overrides, tasks: 1, concurrency: 1, ...replayMasks(source.options.masks, overrides.masks) }, tasks: [{ ...task, ...(observed.length ? { replayFaults: observed.map(fault => ({ ...fault })) } : {}) }], replayOf: { ...ref }, }; diff --git a/tests/restart-chaos/seed.test.ts b/tests/restart-chaos/seed.test.ts index 25e4c348844..20c0437b59e 100644 --- a/tests/restart-chaos/seed.test.ts +++ b/tests/restart-chaos/seed.test.ts @@ -66,3 +66,41 @@ describe("seedChaosLab workspace isolation", () => { expect(calls.some(call => call.method === "POST" && call.route.endsWith("/projects"))).toBe(false); }); }); + +describe("seedChaosLab RC-06 mask", () => { + let fixtureRoot: string; + beforeEach(async () => { + fixtureRoot = await mkdtemp(path.join(os.tmpdir(), "rc06-seed-")); + vi.stubEnv("OPENAI_API_KEY", "test-only-not-a-key"); + }); + afterEach(async () => { + vi.unstubAllEnvs(); + await rm(fixtureRoot, { recursive: true, force: true }); + }); + + const seed = (api: ChaosApi, timeline: Timeline, masks?: string[]) => seedChaosLab({ api, env: "local", adapters: ["codex"], + concurrency: 2, budgetCents: 3_000, fixtureRoot, waveId: "w-test", timeline, ...(masks ? { masks } : {}) }); + const isPolicyCall = (call: Call) => call.method === "POST" && /^\/api\/companies\/[^/]+\/budgets\/policies$/.test(call.route); + + it("keeps the budget amount and disables only the company hard stop", async () => { + const { api, calls } = fakeApi(body => ({ ...body })); + const timeline = new Timeline(null); + await seed(api, timeline, ["rc06"]); + const company = calls.find(call => call.method === "POST" && call.route === "/api/companies")!; + expect((company.body as { budgetMonthlyCents: number }).budgetMonthlyCents).toBe(3_000); + const policies = calls.filter(isPolicyCall); + expect(policies).toHaveLength(1); + expect(policies[0]!.route).toBe("/api/companies/id-1/budgets/policies"); + expect(policies[0]!.body).toEqual({ scopeType: "company", scopeId: "id-1", amount: 3_000, hardStopEnabled: false }); + expect(timeline.entries.filter(entry => entry.kind === "seed_mask_rc06")) + .toEqual([expect.objectContaining({ companyId: "id-1", budgetCents: 3_000, hardStopEnabled: false })]); + }); + + it("leaves the company hard stop on without the mask", async () => { + const { api, calls } = fakeApi(body => ({ ...body })); + const timeline = new Timeline(null); + await seed(api, timeline); + expect(calls.some(isPolicyCall)).toBe(false); + expect(timeline.entries.some(entry => entry.kind === "seed_mask_rc06")).toBe(false); + }); +}); diff --git a/tests/restart-chaos/seed.ts b/tests/restart-chaos/seed.ts index 8b608026414..7cbdfa20f11 100644 --- a/tests/restart-chaos/seed.ts +++ b/tests/restart-chaos/seed.ts @@ -100,6 +100,7 @@ export async function seedChaosLab(input: { adapters: readonly AdapterId[]; concurrency: number; budgetCents: number; + masks?: readonly string[]; fixtureRoot: string; waveId: string; timeline: Timeline; @@ -120,6 +121,13 @@ export async function seedChaosLab(input: { budgetMonthlyCents: input.budgetCents, }); timeline.add("seed_company", { companyId: company.id }); + if (input.masks?.includes("rc06")) { + // Keep the amount, disable only the hard stop on this isolated company (RC-06 mask). + await api.post(`/api/companies/${company.id}/budgets/policies`, { + scopeType: "company", scopeId: company.id, amount: input.budgetCents, hardStopEnabled: false, + }); + timeline.add("seed_mask_rc06", { companyId: company.id, budgetCents: input.budgetCents, hardStopEnabled: false }); + } const secretRefs: SecretReferenceMap = {}; const credentialKeys = [...new Set([...input.adapters.map(adapter => CREDENTIAL_FOR_ADAPTER[adapter]), diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index 1efdf67acd0..ed8cb7e9fcc 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -32,6 +32,13 @@ export interface WaveOptions { daytonaImage?: string; /** `--env daytona`: vCPU per sandbox (see seed.ts chaosDaytonaShape). */ daytonaCpu?: number; + /** + * Known product signatures this wave deliberately masks. "rc06" keeps the + * company budget amount but disables its hard stop, so a stuck + * accounting row (RC-06) cannot wedge every later task. Spend is still + * recorded per wave. Absent on plans written before masks existed. + */ + masks?: string[]; } export interface TaskPlan {