diff --git a/tests/restart-chaos/ttft-baseline-stats.ts b/tests/restart-chaos/ttft-baseline-stats.ts index ce407120494..317a54927d5 100644 --- a/tests/restart-chaos/ttft-baseline-stats.ts +++ b/tests/restart-chaos/ttft-baseline-stats.ts @@ -24,6 +24,9 @@ export interface BaselineRun { export interface BaselineFault { kind: string; readyAt: number; + /** When the fault began. A recovery that a later fault interrupts before its + * next provider event is not a measurement of this fault's recovery. */ + startedAt?: number; /** Runs that were in flight when the fault started. */ runIds: string[]; } @@ -137,7 +140,11 @@ export interface RecoverySample { runId: string; adapter: string; kind: string; * latency when nothing reloads. The judged row. */ controlPlaneMs: number | null; /** The provider's share: session load start to the next provider event. Reported. */ - providerLoadMs: number | null } + providerLoadMs: number | null; + /** A later fault began before this recovery reached its next provider event + * (back-to-back faults). Its latencies are null: the later fault's own sample + * measures that run. */ + interruptedByFault?: true } /** Control plane ready → the recovered run's next provider event, keyed by runner disposition. */ /** The run's trace start: every span records its offset from it. A span event is @@ -180,6 +187,12 @@ export function recoverySamples(runs: readonly BaselineRun[], faults: readonly B && Date.parse(event.createdAt) >= fault.readyAt - 60_000); const next = run.events.find(event => PROVIDER_EVENT_TYPES.has(event.eventType) && Date.parse(event.createdAt) >= fault.readyAt); const nextAt = next ? Date.parse(next.createdAt) : null; + if (nextAt !== null && faults.some(other => other.startedAt !== undefined && other.startedAt > fault.readyAt && other.startedAt < nextAt)) { + samples.push({ runId, adapter: run.adapter, kind: fault.kind, readyAt: fault.readyAt, + disposition: String(transition?.payload?.runnerDisposition ?? "none"), + latencyMs: null, controlPlaneMs: null, providerLoadMs: null, interruptedByFault: true }); + continue; + } const stamped = stampProviderLoadStarts(run); const loadStarts = nextAt === null ? [] : stamped.events.flatMap(event => { if (event.eventType !== "run.performance.span" || event.payload?.span !== "provider.session.load") return []; diff --git a/tests/restart-chaos/wave-speed.test.ts b/tests/restart-chaos/wave-speed.test.ts index c376cd83ee2..2a78be4110e 100644 --- a/tests/restart-chaos/wave-speed.test.ts +++ b/tests/restart-chaos/wave-speed.test.ts @@ -64,6 +64,19 @@ describe("wave speed grade", () => { expect(speed.breaches).toBe(0); }); + it("does not grade a recovery that a back-to-back fault interrupted before its next provider event", () => { + // f1 is ready at 10 s; f2 hits 0.4 s later and is ready at 30 s; the run resumes 1.5 s after f2. + const recovered: BaselineRun = { runId: "r", adapter: "claude", environment: "local", phase: "cold", + events: [{ eventType: "session.resumed", createdAt: new Date(31_500).toISOString() }] }; + const faults = [{ kind: "K4", startedAt: 1_000, readyAt: 10_000, runIds: ["r"] }, + { kind: "K6", startedAt: 10_400, readyAt: 30_000, runIds: ["r"] }]; + const windows = outageWindows(faults.map(fault => ({ startedAt: fault.startedAt, readyAt: fault.readyAt }) as FaultReceipt)); + const speed = gradeWaveSpeed("local", [recovered], faults, 0, [], { adjusted: 0, removedMs: 0, windows }); + expect(speed.rows.find(row => row.metric === "restart_recovery_control_plane")).toMatchObject({ measured: { n: 1, p50: 1_500 }, breaches: [] }); + expect(speed.recoveryInterruptedByFault).toBe(1); + expect(speed.recoveryWithoutProviderEvent).toBe(0); + }); + it("removes a later outage from restart recovery", () => { const recovered: BaselineRun = { runId: "r", adapter: "claude", environment: "local", phase: "cold", events: [{ eventType: "turn.started", createdAt: new Date(10_000 + 302_000).toISOString() }] }; @@ -183,6 +196,6 @@ describe("wave speed grade", () => { const fault = { kind: "K2", startedAt: 1_000, readyAt: 2_000 } as FaultReceipt; expect(faultRuns([fault], [{ id: "before", createdAt: 0, finishedAt: 500 }, { id: "live", createdAt: 0, finishedAt: null }, { id: "during", createdAt: 1_500, finishedAt: null }, { id: "after", createdAt: 2_500, finishedAt: null }])) - .toEqual([{ kind: "K2", readyAt: 2_000, runIds: ["live", "during"] }]); + .toEqual([{ kind: "K2", readyAt: 2_000, startedAt: 1_000, runIds: ["live", "during"] }]); }); }); diff --git a/tests/restart-chaos/wave-speed.ts b/tests/restart-chaos/wave-speed.ts index c57973fa00b..065fd1d24ce 100644 --- a/tests/restart-chaos/wave-speed.ts +++ b/tests/restart-chaos/wave-speed.ts @@ -22,6 +22,8 @@ export interface WaveSpeed { outageAdjustedSpans: number; outageRemovedMs: number; recoveryWithoutProviderEvent: number; + /** Recovery samples a later fault interrupted before their next provider event (not graded). */ + recoveryInterruptedByFault: number; runBudgetExceeded: number; singleRunBreaches: number; breaches: number; @@ -53,7 +55,7 @@ export function breachesOf(measured: Distribution, budget: SpeedBudget | null): } export function faultRuns(faults: readonly FaultReceipt[], runs: ReadonlyArray<{ id: string; createdAt: number; finishedAt: number | null }>): BaselineFault[] { - return faults.filter(fault => fault.readyAt !== null).map(fault => ({ kind: fault.kind, readyAt: fault.readyAt!, + return faults.filter(fault => fault.readyAt !== null).map(fault => ({ kind: fault.kind, readyAt: fault.readyAt!, startedAt: fault.startedAt, runIds: runs.filter(run => run.createdAt <= fault.readyAt! && (run.finishedAt === null || run.finishedAt >= fault.startedAt)) .map(run => run.id) })); } @@ -211,7 +213,9 @@ export function gradeWaveSpeed(environment: EnvironmentKind, runs: readonly Base || (boot !== null && bootBudget !== null && boot > bootBudget.p95Ms); }).length; const breaches = rows.reduce((total, row) => total + row.breaches.length, 0) + singleRunBreaches; - return { environment, rows, recoveryWithoutProviderEvent: samples.filter(sample => sample.latencyMs === null).length, + return { environment, rows, + recoveryWithoutProviderEvent: samples.filter(sample => sample.latencyMs === null && !sample.interruptedByFault).length, + recoveryInterruptedByFault: samples.filter(sample => sample.interruptedByFault).length, runBudgetExceeded, singleRunBreaches, breaches, errors, outageAdjustedSpans: outage.adjusted, outageRemovedMs: outage.removedMs }; } @@ -282,7 +286,7 @@ export function speedMarkdown(speed: WaveSpeed): string { const s = (value: number | null) => value === null ? "—" : `${(value / 1000).toFixed(1)} s`; const lines = [`## Speed (${speed.environment})`, "", `Breaches: ${speed.breaches} (aggregate p50/p95 rows plus ${speed.singleRunBreaches} runs over the p95 budget after outage removal; the server logged ${speed.runBudgetExceeded} budget_exceeded events before removal).`, - `Restart samples with no provider event after ready: ${speed.recoveryWithoutProviderEvent}.`, + `Restart samples with no provider event after ready: ${speed.recoveryWithoutProviderEvent}. Interrupted by a later fault (not graded): ${speed.recoveryInterruptedByFault}.`, `Outage time removed from ${speed.outageAdjustedSpans} spans: ${s(speed.outageRemovedMs)} (fault start to server ready; budgets judge the rest).`, "", "| Adapter | Metric | Start | n | p50 | p95 | Budget p50 / p95 | Breach |", "|---|---|---|---|---|---|---|---|"]; for (const row of speed.rows) {