diff --git a/tests/restart-chaos/wave-speed.test.ts b/tests/restart-chaos/wave-speed.test.ts index 9e2ef2f232a..e14a56440ec 100644 --- a/tests/restart-chaos/wave-speed.test.ts +++ b/tests/restart-chaos/wave-speed.test.ts @@ -1,7 +1,8 @@ import { describe, expect, it } from "vitest"; import type { BaselineRun } from "./ttft-baseline-stats.js"; +import type { ChaosApi } from "./api.js"; import type { FaultReceipt } from "./types.js"; -import { breachesOf, budgetFor, faultRuns, firstActivityMs, gradeWaveSpeed, outageWindows, speedMarkdown, traceStartMs, withoutOutage } from "./wave-speed.js"; +import { breachesOf, budgetFor, collectWaveSpeed, faultBeforeFirstActivity, faultRuns, firstActivityMs, gradeWaveSpeed, outageWindows, speedMarkdown, traceStartMs, withoutOutage } from "./wave-speed.js"; import { runMetrics } from "./ttft-baseline-stats.js"; const span = (name: string, startOffsetMs: number, durationMs: number, extra: Record = {}) => @@ -68,6 +69,25 @@ describe("wave speed grade", () => { createdAt: new Date(trace + offset + duration).toISOString(), payload: { span: name, startOffsetMs: offset, durationMs: duration } })), }); + it("treats a run that a fault hit before its first activity as a restart, not a cold start", async () => { + const hit = traced(1_000_000, [["heartbeat.queue", 0, 100], ["task.time_to_first_agent_activity", 0, 30_000]]); + expect(faultBeforeFirstActivity(hit, [1_010_000], 1_000_000)).toBe(true); + expect(faultBeforeFirstActivity(hit, [1_040_000, 990_000], 1_000_000)).toBe(false); + expect(faultBeforeFirstActivity(hit, [1_005_000], 1_008_000)).toBe(false); + const clean = traced(2_000_000, [["heartbeat.queue", 0, 100], ["task.time_to_first_agent_activity", 0, 4_000]]); + const events: Record = { hit: [...hit.events, { eventType: "turn.started", createdAt: new Date(1_013_000).toISOString() }], + clean: clean.events }; + const api = { get: async (route: string) => (route.includes("/hit/") ? events.hit : events.clean) + .map((event: object, seq: number) => ({ ...event, seq })) } as unknown as ChaosApi; + const sample = (id: string, createdAt: number) => ({ runs: [{ id, status: "succeeded", createdAt: new Date(createdAt).toISOString(), + finishedAt: new Date(createdAt + 60_000).toISOString() }] }) as never; + const fault = { kind: "K1", startedAt: 1_010_000, readyAt: 1_012_000 } as FaultReceipt; + const speed = await collectWaveSpeed(api, "local", [{ issueId: "i1", adapter: "claude", samples: [sample("hit", 1_000_000)] }, + { issueId: "i2", adapter: "claude", samples: [sample("clean", 2_000_000)] }], [fault]); + expect(speed.rows.find(row => row.metric === "time_to_first_activity" && row.start === "cold")).toMatchObject({ measured: { n: 1, p50: 4_000 } }); + expect(speed.rows.find(row => row.metric === "restart_recovery")).toMatchObject({ adapter: "claude", measured: { n: 1 } }); + }); + it("finds the trace start from when its spans were written, never later than it", () => { expect(traceStartMs(traced(1_000_000, [["a", 0, 5_000], ["b", 2_000, 1_000]]))).toBe(1_000_000); const late = traced(1_000_000, [["a", 0, 5_000]]); diff --git a/tests/restart-chaos/wave-speed.ts b/tests/restart-chaos/wave-speed.ts index bfa77ce0d03..2c0cb82c831 100644 --- a/tests/restart-chaos/wave-speed.ts +++ b/tests/restart-chaos/wave-speed.ts @@ -100,6 +100,16 @@ export function withoutOutage(run: BaselineRun, windows: readonly OutageWindow[] return { run: { ...run, events }, adjusted, removedMs, anchored: true }; } +export function faultBeforeFirstActivity(run: BaselineRun, faultStarts: readonly number[], runCreatedAt: number): boolean { + const traceStart = traceStartMs(run); + if (traceStart === null) return false; + const spans = runSpans(run); + const first = spans.find(span => span.name === "task.time_to_first_agent_activity") + ?? spans.find(span => span.name === "task.time_to_first_token"); + const end = first ? traceStart + first.startOffsetMs + first.durationMs : Number.POSITIVE_INFINITY; + return faultStarts.some(at => at >= runCreatedAt && at <= end); +} + export function firstActivityMs(run: BaselineRun): number | null { const metrics = runMetrics(run); return metrics.firstActivityAfterRunCreatedMs ?? metrics.ttftAfterRunCreatedMs; @@ -188,6 +198,8 @@ export async function collectWaveSpeed(api: ChaosApi, environment: EnvironmentKi try { const raw: BaselineRun = { runId: run.id, adapter: task.adapter, environment, phase: index === 0 ? "cold" : "warm", events: await runEventsWithRetry(api, run.id) }; + if (faultBeforeFirstActivity(raw, faults.filter(fault => fault.readyAt !== null).map(fault => fault.startedAt), + Date.parse(run.createdAt!))) raw.phase = "restart"; const trimmed = withoutOutage(raw, windows); outage.adjusted += trimmed.adjusted; outage.removedMs += trimmed.removedMs; runs.push(trimmed.run);