diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index c4c36f9900a..ba83df15fd7 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -406,7 +406,14 @@ part of U3 is checked only in waves run with this harness. - R1 allows the workload budget plus outage time. When a Retry was needed R1 is `n/a` and R3 decides; a task that rests unfinished with no Retry fails both. A task that never rested within its deadline fails R1 as `no_rest_within_budget`, - whatever its last status. + whatever its last status. R1 also allows the first run's initial queue wait + (minus outage time inside it) when `wake-evidence/agent-slots.json` proves it + fair: no newer fresh assignment run of that agent started during the wait + (newer follow-ups and retries rank higher by design), and the agent had no + idle slot for more than 2 s outside a server outage. A provably unfair wait + fails as `unfair_queue_wait`; one that cannot be proven is not credited, and + the R1 detail says why. + Waits of later runs (recovery, restarts, retries) are never credited. - Speed: an aggregate p50/p95 row can breach only with at least 5 samples; a smaller row is reported as `insufficient sample (n < 5)`. Every cold or warm run over its p95 budget, and every restart recovery whose Paperclip share exceeds diff --git a/tests/restart-chaos/agent-slots.ts b/tests/restart-chaos/agent-slots.ts index a13ed7a9c9e..b5f53264b64 100644 --- a/tests/restart-chaos/agent-slots.ts +++ b/tests/restart-chaos/agent-slots.ts @@ -4,7 +4,56 @@ export const AGENT_SLOTS_FILE = "agent-slots.json"; export interface AgentSlots { agents: Array<{ agentId: string; maxConcurrentRuns: number | null }>; - runs: Array<{ runId: string; agentId: string; startedAt: number | null; finishedAt: number | null }>; + runs: Array<{ runId: string; agentId: string; createdAt?: number | null; invocationSource?: string | null; retryOfRunId?: string | null; + startedAt: number | null; finishedAt: number | null }>; +} + +export const QUEUE_HANDOFF_TOLERANCE_MS = 2_000; + +export type QueueFairness = { verdict: "fair" } | { verdict: "unfair" | "unproven"; reason: string }; +export type QueueWaitFairness = (runId: string, createdAt: number, startedAt: number, outages?: ReadonlyArray) => QueueFairness; + +const iso = (at: number) => new Date(at).toISOString(); + +export function queueWaitFairnessFrom(slots: AgentSlots | null): QueueWaitFairness { + const caps = new Map((slots?.agents ?? []).map(agent => [agent.agentId, agent.maxConcurrentRuns])); + const agentOf = new Map((slots?.runs ?? []).map(run => [run.runId, run.agentId])); + return (runId, createdAt, startedAt, outages = []) => { + const agentId = agentOf.get(runId); + const cap = agentId ? caps.get(agentId) : undefined; + if (!slots || !agentId || typeof cap !== "number" || cap < 1) return { verdict: "unproven", reason: `no agent slot evidence for ${runId}` }; + const others = slots.runs.filter(run => run.agentId === agentId && run.runId !== runId); + const malformed = others.filter(run => [run.startedAt, run.finishedAt].some(at => at !== null && !Number.isFinite(at))); + if (malformed.length) return { verdict: "unproven", reason: `malformed start or finish time for ${malformed.map(run => run.runId).join(",")}` }; + const startedAhead = others.filter(run => Number.isFinite(run.startedAt) && run.startedAt! >= createdAt && run.startedAt! < startedAt); + const newer = startedAhead.filter(run => Number.isFinite(run.createdAt) && run.createdAt! > createdAt); + const overtaker = newer.find(run => run.invocationSource === "assignment" && !run.retryOfRunId); + if (overtaker) { + return { verdict: "unfair", reason: `assignment run ${overtaker.runId} created ${iso(overtaker.createdAt!)} started ${iso(overtaker.startedAt!)} ahead of ${runId}` }; + } + const running = (at: number) => others.filter(run => run.startedAt !== null && run.startedAt <= at + && (run.finishedAt === null || run.finishedAt > at)).length; + const points = [...new Set([createdAt, startedAt, ...others.flatMap(run => [run.startedAt, run.finishedAt]) + .filter((at): at is number => at !== null && at > createdAt && at < startedAt)])].sort((a, b) => a - b); + const idleGaps: Array<[number, number]> = []; + let idleFrom: number | null = null; + for (const at of points) { + if (at < startedAt && running(at) < cap) { idleFrom ??= at; continue; } + if (idleFrom !== null && at - idleFrom > QUEUE_HANDOFF_TOLERANCE_MS) idleGaps.push([idleFrom, at]); + idleFrom = null; + } + const duringOutage = ([from, to]: readonly [number, number]) => outages.some(([start, end]) => start < to && end > from); + const idle = idleGaps.find(gap => !duringOutage(gap)); + if (idle) return { verdict: "unfair", reason: `agent ${agentId} had an idle slot from ${iso(idle[0])} to ${iso(idle[1])} while ${runId} was queued` }; + if (idleGaps.length) { + return { verdict: "unproven", reason: `idle slot from ${iso(idleGaps[0]![0])} to ${iso(idleGaps[0]![1])} overlapped a server outage while ${runId} was queued` }; + } + const unknown = startedAhead.filter(run => !Number.isFinite(run.createdAt)); + if (unknown.length) return { verdict: "unproven", reason: `no createdAt for ${unknown.map(run => run.runId).join(",")}, which started while ${runId} was queued` }; + const unclassified = newer.filter(run => run.invocationSource == null); + if (unclassified.length) return { verdict: "unproven", reason: `no invocation source for newer run ${unclassified.map(run => run.runId).join(",")}, which started while ${runId} was queued` }; + return { verdict: "fair" }; + }; } export function slotFreeFrom(slots: AgentSlots | null): SlotFree { @@ -30,6 +79,8 @@ export function agentSlotsFromRows(agents: ReadonlyArray const cap = Number(agent.maxConcurrentRuns); return { agentId: String(agent.id), maxConcurrentRuns: Number.isInteger(cap) && cap > 0 ? cap : null }; }), - runs: runs.filter(run => run.agentId).map(run => ({ runId: String(run.id), agentId: String(run.agentId), startedAt: ms(run.startedAt), finishedAt: ms(run.finishedAt) })), + runs: runs.filter(run => run.agentId).map(run => ({ runId: String(run.id), agentId: String(run.agentId), createdAt: ms(run.createdAt), + invocationSource: run.invocationSource == null ? null : String(run.invocationSource), retryOfRunId: run.retryOfRunId == null ? null : String(run.retryOfRunId), + startedAt: ms(run.startedAt), finishedAt: ms(run.finishedAt) })), }; } diff --git a/tests/restart-chaos/grader.test.ts b/tests/restart-chaos/grader.test.ts index f6495d49e68..b1b63e77195 100644 --- a/tests/restart-chaos/grader.test.ts +++ b/tests/restart-chaos/grader.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; -import { crossingFaults, gradeR2, gradeR3, gradeR4, gradeR5, gradeTask, STALL_LIMIT_MS } from "./grader.js"; +import { QUEUE_HANDOFF_TOLERANCE_MS, queueWaitFairnessFrom, type AgentSlots } from "./agent-slots.js"; +import { crossingFaults, gradeR1, gradeR2, gradeR3, gradeR4, gradeR5, gradeTask, STALL_LIMIT_MS } from "./grader.js"; import { isResting } from "./observer.js"; import type { FaultReceipt, OracleArtifacts, RunSample, TaskRecord, TaskSample } from "./types.js"; import { BENCH_EXPECTED, parseBenchRecord, r1TestChecksSlug, stallWakeChecks, workloadDefinitions } from "./workloads.js"; @@ -191,6 +192,129 @@ describe("R1–R6 grading on fixture timelines", () => { }); }); +describe("R1 fair queue wait credit", () => { + const BUDGET = 2_200_000; + const WAIT = 1_509_000; + const CREATED = 50; + const STARTED = CREATED + WAIT; + type SlotRun = AgentSlots["runs"][number]; + const slotRun = (runId: string, created: number, started: number, finished: number | null): SlotRun => + ({ runId, agentId: "agent-oc", createdAt: T0 + created, invocationSource: "assignment", retryOfRunId: null, startedAt: T0 + started, + finishedAt: finished === null ? null : T0 + finished }); + const own = (started: number = STARTED): SlotRun => ({ runId: "r1", agentId: "agent-oc", createdAt: T0 + CREATED, startedAt: T0 + started, finishedAt: null }); + const fullAgent = (): SlotRun[] => [ + slotRun("o1", -900_000, -800_000, 400_000), + slotRun("o2", -900_000, -700_000, 900_000), + slotRun("o3", -900_000, -600_000, STARTED), + slotRun("o4", -900_000, -500_000, 2_000_000), + slotRun("o5", 10, 400_000, 2_100_000), + slotRun("o6", 20, 901_000, 2_100_000), + ]; + const slots = (runs: SlotRun[]): AgentSlots => ({ agents: [{ agentId: "agent-oc", maxConcurrentRuns: 4 }], runs: [own(), ...runs] }); + const queued = (tookMs: number, wait = WAIT) => record([ + sample(60_000, "in_progress", [run("r1", "queued", CREATED, null, { startedAt: null })]), + sample(CREATED + wait + 5_000, "in_progress", [run("r1", "running", CREATED, null, { startedAt: iso(CREATED + wait) })]), + sample(tookMs, "done", [run("r1", "succeeded", CREATED, tookMs - 1_000, { startedAt: iso(CREATED + wait) })]), + ], { restAt: T0 + tookMs, faultIds: [] }); + const t031 = () => queued(BUDGET + 87_000); + + it("credits a fair 1,509 s FIFO queue wait that left the task 87 s over the old budget", () => { + expect(gradeR1(t031(), BUDGET, [])).toEqual({ verdict: "fail", detail: "done_too_late 2287s > 2200s (queue wait not credited: no agent slot evidence for r1)" }); + const graded = gradeR1(t031(), BUDGET, [], queueWaitFairnessFrom(slots(fullAgent()))); + expect(graded).toEqual({ verdict: "pass", detail: "done after 2287s (allowed 3709s incl. 1509s fair queue wait)" }); + }); + + it("fails as an unfair queue wait when a newer run started ahead of the queued run", () => { + const runs = fullAgent().map(entry => entry.runId === "o6" ? { ...entry, createdAt: T0 + 300_000 } : entry); + const graded = gradeR1(t031(), BUDGET, [], queueWaitFairnessFrom(slots(runs))); + expect(graded.verdict).toBe("fail"); + expect(graded.detail).toMatch(/^unfair_queue_wait assignment run o6 created .* ahead of r1/); + const task = gradeTask(t031(), [], BUDGET, oracle(), undefined, { queueWait: queueWaitFairnessFrom(slots(runs)) }); + expect(task.signature).toBe("R1:unfair_queue_wait:A1:K1"); + }); + + it("fails as an unfair queue wait when the agent had an idle slot during the wait", () => { + const runs = fullAgent().map(entry => entry.runId === "o6" ? { ...entry, startedAt: T0 + 931_000 } : entry); + const graded = gradeR1(t031(), BUDGET, [], queueWaitFairnessFrom(slots(runs))); + expect(graded.verdict).toBe("fail"); + expect(graded.detail).toContain(`idle slot from ${iso(900_000)} to ${iso(931_000)} while r1 was queued`); + }); + + it("credits the wait when a newer follow-up or retry that the dispatcher ranks higher started ahead", () => { + const followUp = fullAgent().map(entry => entry.runId === "o6" ? { ...entry, createdAt: T0 + 300_000, invocationSource: "on_demand" } : entry); + expect(gradeR1(t031(), BUDGET, [], queueWaitFairnessFrom(slots(followUp))).verdict).toBe("pass"); + const retry = fullAgent().map(entry => entry.runId === "o6" ? { ...entry, createdAt: T0 + 300_000, retryOfRunId: "o2" } : entry); + expect(gradeR1(t031(), BUDGET, [], queueWaitFairnessFrom(slots(retry))).verdict).toBe("pass"); + for (const invocationSource of [undefined, null]) { + const unclassified = fullAgent().map(entry => entry.runId === "o6" ? { ...entry, createdAt: T0 + 300_000, invocationSource } : entry); + expect(gradeR1(t031(), BUDGET, [], queueWaitFairnessFrom(slots(unclassified))).detail) + .toBe("done_too_late 2287s > 2200s (queue wait not credited: no invocation source for newer run o6, which started while r1 was queued)"); + } + const malformed = fullAgent().map(entry => entry.runId === "o6" ? { ...entry, createdAt: Number.NaN } : entry); + expect(gradeR1(t031(), BUDGET, [], queueWaitFairnessFrom(slots(malformed))).detail) + .toBe("done_too_late 2287s > 2200s (queue wait not credited: no createdAt for o6, which started while r1 was queued)"); + for (const field of ["startedAt", "finishedAt"] as const) { + const badTime = fullAgent().map(entry => entry.runId === "o2" ? { ...entry, [field]: Number.NaN } : entry); + expect(gradeR1(t031(), BUDGET, [], queueWaitFairnessFrom(slots(badTime))).detail) + .toBe("done_too_late 2287s > 2200s (queue wait not credited: malformed start or finish time for o2)"); + } + }); + + it("does not call an idle slot during a server outage unfair", () => { + const runs = fullAgent().map(entry => entry.runId === "o6" ? { ...entry, startedAt: T0 + 931_000 } : entry); + const outage: FaultReceipt = { ...fault, startedAt: T0 + 890_000, readyAt: T0 + 930_000 }; + const graded = gradeR1(t031(), BUDGET, [outage], queueWaitFairnessFrom(slots(runs))); + expect(graded.verdict).toBe("fail"); + expect(graded.detail).toMatch(/^done_too_late .*queue wait not credited: idle slot from .* overlapped a server outage while r1 was queued/); + }); + + it("allows a same-second slot hand-off within the tolerance", () => { + const runs = fullAgent().map(entry => entry.runId === "o6" ? { ...entry, startedAt: T0 + 900_000 + QUEUE_HANDOFF_TOLERANCE_MS } : entry); + expect(gradeR1(t031(), BUDGET, [], queueWaitFairnessFrom(slots(runs))).verdict).toBe("pass"); + }); + + it("does not credit a wait it cannot prove fair", () => { + expect(gradeR1(t031(), BUDGET, [], queueWaitFairnessFrom(null)).detail) + .toBe("done_too_late 2287s > 2200s (queue wait not credited: no agent slot evidence for r1)"); + const runs = fullAgent().map(entry => entry.runId === "o6" ? { ...entry, createdAt: null } : entry); + expect(gradeR1(t031(), BUDGET, [], queueWaitFairnessFrom(slots(runs))).detail) + .toBe("done_too_late 2287s > 2200s (queue wait not credited: no createdAt for o6, which started while r1 was queued)"); + }); + + it("says so when the first run never started", () => { + const cancelled = record([sample(BUDGET + 87_000, "done", [ + run("r0", "cancelled", 0, 1_000, { startedAt: null }), + run("r1", "succeeded", CREATED, BUDGET, { startedAt: iso(STARTED) }), + ])], { restAt: T0 + BUDGET + 87_000, faultIds: [] }); + expect(gradeR1(cancelled, BUDGET, [], queueWaitFairnessFrom(slots(fullAgent()))).detail) + .toBe("done_too_late 2287s > 2200s (queue wait not credited: first run r0 never started)"); + }); + + it("leaves a short queue wait unchanged", () => { + const wait = 3_000; + const runs = [slotRun("o1", -900_000, -800_000, CREATED + wait), slotRun("o2", 10, 1_000, 900_000)]; + const fairness = queueWaitFairnessFrom({ agents: [{ agentId: "agent-oc", maxConcurrentRuns: 4 }], runs: [own(CREATED + wait), ...runs] }); + const inBudget = queued(BUDGET - 10_000, wait); + expect(gradeR1(inBudget, BUDGET, [], fairness)).toEqual(gradeR1(inBudget, BUDGET, [])); + expect(gradeR1(inBudget, BUDGET, [], fairness)).toEqual({ verdict: "pass", detail: "done after 2190s (allowed 2200s)" }); + expect(gradeR1(queued(BUDGET + 87_000, wait), BUDGET, [], fairness).verdict).toBe("fail"); + }); + + it("credits only the first run's initial wait, not outage time inside it or later retries", () => { + const outage: FaultReceipt = { ...fault, startedAt: T0 + 100_000, readyAt: T0 + 200_000 }; + const graded = gradeR1(queued(BUDGET + 1_550_000), BUDGET, [outage], queueWaitFairnessFrom(slots(fullAgent()))); + expect(graded).toEqual({ verdict: "fail", detail: "done_too_late 3750s > 3709s incl. 1409s fair queue wait" }); + const retried = record([sample(BUDGET + 87_000, "done", [ + run("r0", "failed", 0, 1_000), + run("r1", "succeeded", CREATED, BUDGET, { startedAt: iso(STARTED), retryOfRunId: "r0" }), + ])], { restAt: T0 + BUDGET + 87_000, faultIds: [] }); + const retriedGrade = gradeR1(retried, BUDGET, [], queueWaitFairnessFrom(slots(fullAgent()))); + expect(retriedGrade.verdict).toBe("fail"); + expect(retriedGrade.detail).toMatch(/^done_too_late 2287s > 2200s/); + expect(retriedGrade.detail).not.toContain("fair queue wait"); + }); +}); + describe("T-oracles", () => { it("grades M1 delegation: two done subtasks, part 1 delegated, and a summary of both", () => { const agents = { self: { id: "self", name: "Chaos codex" }, other: { id: "other", name: "Chaos claude" } }; diff --git a/tests/restart-chaos/grader.ts b/tests/restart-chaos/grader.ts index 1c7ad539cb7..e25252954a9 100644 --- a/tests/restart-chaos/grader.ts +++ b/tests/restart-chaos/grader.ts @@ -1,6 +1,7 @@ import { UI_RETRY_BLOCKED_BY_MODAL, UI_RETRY_BUTTON_MISSING, UI_RETRY_NO_REQUEST, type FaultReceipt, type OracleResult, type RuleId, type RuleResult, type TaskGrade, type TaskRecord, type TaskSample } from "./types.js"; import { g1Decision } from "./workloads.js"; import { gradeDiskFullRetry, isDiskFullFault } from "./disk-full-faults.js"; +import { queueWaitFairnessFrom, type QueueFairness, type QueueWaitFairness } from "./agent-slots.js"; import { enforcedUniversalFailures, gradeUniversal, type SlotFree, type UniversalBounds, type UniversalDbEvidence } from "./universal.js"; /** Pure R1–R6 grading over observer samples (plan §2). No I/O. */ @@ -62,7 +63,37 @@ export function expectedFinalStatus(record: TaskRecord): string { return record.plan.workload === "G1" && g1Decision(record.plan.nonce) === "reject" ? "cancelled" : "done"; } -export function gradeR1(record: TaskRecord, budgetMs: number, faults: readonly FaultReceipt[]): RuleResult { +export function firstRun(record: TaskRecord): NonNullable[number] | null { + const latest = new Map[number]>(); + for (const sample of record.samples) for (const run of sample.runs ?? []) latest.set(run.id, run); + const first = [...latest.values()].filter(run => Number.isFinite(ms(run.createdAt))) + .sort((a, b) => ms(a.createdAt) - ms(b.createdAt))[0]; + return first && !first.retryOfRunId ? first : null; +} + +export function firstRunQueuedMs(record: TaskRecord, now: number): number { + const first = firstRun(record); + return first && first.status === "queued" && !first.startedAt ? Math.max(0, now - ms(first.createdAt)) : 0; +} + +export function firstRunQueueWait(record: TaskRecord): { runId: string; createdAt: number; startedAt: number } | null { + const first = firstRun(record); + if (!first || !Number.isFinite(ms(first.startedAt))) return null; + const createdAt = ms(first.createdAt); + const startedAt = ms(first.startedAt); + return startedAt > createdAt ? { runId: first.id, createdAt, startedAt } : null; +} + +function faultOverlapMs(faults: readonly FaultReceipt[], from: number, to: number): number { + return faults.reduce((total, fault) => { + const [start, stop] = faultWindow(fault); + const overlap = Math.min(stop, to) - Math.max(start, from); + return total + (Number.isFinite(overlap) && overlap > 0 ? overlap : 0); + }, 0); +} + +export function gradeR1(record: TaskRecord, budgetMs: number, faults: readonly FaultReceipt[], + queueWait: QueueWaitFairness = queueWaitFairnessFrom(null)): RuleResult { if (record.launchError) return { verdict: "fail", detail: `launch_failed: ${record.launchError}` }; if (record.retry) return { verdict: "n/a", detail: "task rested needing Retry; graded under R3" }; const expected = expectedFinalStatus(record); @@ -70,11 +101,23 @@ export function gradeR1(record: TaskRecord, budgetMs: number, faults: readonly F if (record.finalStatus !== expected) { return { verdict: "fail", detail: `rested_not_done (status ${record.finalStatus ?? "unknown"}, no Retry offered)` }; } - const allowed = budgetMs + outageOverlapMs(record, faults); + const base = budgetMs + outageOverlapMs(record, faults); const took = (record.restAt ?? Number.POSITIVE_INFINITY) - record.createdAt!; + const seconds = (value: number) => Math.round(value / 1000); + if (took <= base) return { verdict: "pass", detail: `${expected} after ${seconds(took)}s (allowed ${seconds(base)}s)` }; + const first = firstRun(record); + const wait = firstRunQueueWait(record); + const fairness: QueueFairness | null = wait ? queueWait(wait.runId, wait.createdAt, wait.startedAt, faults.map(faultWindow)) + : first && !first.startedAt ? { verdict: "unproven", reason: `first run ${first.id} never started` } : null; + if (fairness?.verdict === "unfair") { + return { verdict: "fail", detail: `unfair_queue_wait ${fairness.reason}; done ${seconds(took)}s > ${seconds(base)}s without queue credit` }; + } + const credit = wait && fairness?.verdict === "fair" ? Math.max(0, wait.startedAt - wait.createdAt - faultOverlapMs(faults, wait.createdAt, wait.startedAt)) : 0; + const allowed = base + credit; + const note = credit ? ` incl. ${seconds(credit)}s fair queue wait` : fairness?.verdict === "unproven" ? ` (queue wait not credited: ${fairness.reason})` : ""; return took <= allowed - ? { verdict: "pass", detail: `${expected} after ${Math.round(took / 1000)}s (allowed ${Math.round(allowed / 1000)}s)` } - : { verdict: "fail", detail: `done_too_late ${Math.round(took / 1000)}s > ${Math.round(allowed / 1000)}s` }; + ? { verdict: "pass", detail: `${expected} after ${seconds(took)}s (allowed ${seconds(allowed)}s${note})` } + : { verdict: "fail", detail: `done_too_late ${seconds(took)}s > ${seconds(allowed)}s${note}` }; } export function gradeR2(record: TaskRecord, oracle: OracleResult): RuleResult { @@ -253,10 +296,10 @@ export function gradeR6(record: TaskRecord): RuleResult { } export function gradeTask(record: TaskRecord, faults: readonly FaultReceipt[], budgetMs: number, oracle: OracleResult, - universalDb?: UniversalDbEvidence, queue: { bounds?: UniversalBounds; slotFree?: SlotFree } = {}): TaskGrade { + universalDb?: UniversalDbEvidence, queue: { bounds?: UniversalBounds; slotFree?: SlotFree; queueWait?: QueueWaitFairness } = {}): TaskGrade { const crossing = crossingFaults(record, faults); const rules: Record = { - R1: gradeR1(record, budgetMs, faults), + R1: gradeR1(record, budgetMs, faults, queue.queueWait), R2: gradeR2(record, oracle), R3: gradeR3(record), R4: gradeR4(record), diff --git a/tests/restart-chaos/observer.test.ts b/tests/restart-chaos/observer.test.ts index 64f031617ed..a4364cdb812 100644 --- a/tests/restart-chaos/observer.test.ts +++ b/tests/restart-chaos/observer.test.ts @@ -109,6 +109,39 @@ describe("TaskObserver", () => { expect(observer.done).toBe(true); }); + it("extends the deadline once by the first run's queue wait when it starts", async () => { + let createdAt = 0; + let startedAt: number | null = null; + const { api } = fakeApi({ "/runs": () => [{ runId: "r1", status: startedAt === null ? "queued" : "running", + createdAt: new Date(createdAt).toISOString(), startedAt: startedAt === null ? null : new Date(startedAt).toISOString() }] }, "in_progress"); + const observer = new TaskObserver({ ...plan, workload: "A1", followUpAfterMs: null }, api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + const deadline = observer.record.deadlineAt!; + createdAt = start; + await observer.poll(start + 1_000); + expect(observer.record.deadlineAt).toBe(deadline); + startedAt = start + 600_000; + await observer.poll(start + 601_000); + expect(observer.record.deadlineAt).toBe(deadline + 600_000); + await observer.poll(start + 602_000); + expect(observer.record.deadlineAt).toBe(deadline + 600_000); + }); + + it("does not time out while the first run is still queued, up to one more budget", async () => { + let createdAt = 0; + const { api } = fakeApi({ "/runs": () => [{ runId: "r1", status: "queued", createdAt: new Date(createdAt).toISOString(), startedAt: null }] }, "in_progress"); + const observer = new TaskObserver({ ...plan, workload: "A1", followUpAfterMs: null }, api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + const budget = observer.record.deadlineAt! - start; + createdAt = start; + await observer.poll(start + budget + 1_000); + expect(observer.record.timedOut).toBe(false); + await observer.poll(start + 2 * budget + 1_000); + expect(observer.record.timedOut).toBe(true); + }); + it("keeps watching unexplained idle work until the stall limit, then records the stall", async () => { const { api } = fakeApi({}, "in_progress"); const observer = new TaskObserver({ ...plan, workload: "A1", followUpAfterMs: null }, api, lab, new Timeline(null), ready); diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index 2bfff8e6e9c..8c43a078d25 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -6,7 +6,7 @@ import os from "node:os"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { ChaosApiError, type ChaosApi } from "./api.js"; -import { ACTIVE_RUN_STATUSES, isStalledSample, STALL_LIMIT_MS } from "./grader.js"; +import { ACTIVE_RUN_STATUSES, firstRunQueuedMs, firstRunQueueWait, isStalledSample, STALL_LIMIT_MS } from "./grader.js"; import type { ChaosLab } from "./seed.js"; import type { Timeline } from "./timeline.js"; import type { @@ -218,6 +218,7 @@ export class TaskObserver { private stallSince: number | null = null; /** First sample of the current stretch with the server ready (H1 holds the pause across it). */ private readySince: number | null = null; + private queueWaitExtended = false; private readonly faultSchedule: ScheduledFault[]; private readonly pendingFaults: ScheduledFault[]; done = false; @@ -491,10 +492,19 @@ export class TaskObserver { /** Outages extend the deadline: R1 allows budget plus outage time. */ extendDeadline(ms: number) { if (this.record.deadlineAt !== null) this.record.deadlineAt += ms; } + private extendForFirstQueueWait() { + if (this.queueWaitExtended) return; + const wait = firstRunQueueWait(this.record); + if (!wait) return; + this.queueWaitExtended = true; + this.extendDeadline(wait.startedAt - wait.createdAt); + } + async poll(now = Date.now()): Promise { if (this.done || !this.record.issueId) return; const sample = await this.sample(now); this.record.samples.push(sample); + this.extendForFirstQueueWait(); await this.captureRemoteFilesAtDone(sample); this.readySince = sample.reachable && sample.serverReady ? (this.readySince ?? now) : null; await this.observeBenchProgress(); @@ -542,7 +552,8 @@ export class TaskObserver { if (this.done) return; } } - if (this.record.deadlineAt !== null && now > this.record.deadlineAt && !this.supervisor.faultInProgress) { + const queuedMs = this.queueWaitExtended ? 0 : Math.min(firstRunQueuedMs(this.record, now), this.definition.budgetMs); + if (this.record.deadlineAt !== null && now > this.record.deadlineAt + queuedMs && !this.supervisor.faultInProgress) { if (this.restCandidateSince !== null && this.restCandidateSince <= this.record.deadlineAt && now - this.restCandidateSince < REST_CONFIRM_MS) return; this.record.timedOut = true; diff --git a/tests/restart-chaos/wake-evidence.test.ts b/tests/restart-chaos/wake-evidence.test.ts index 686c7d2e799..cea83c75142 100644 --- a/tests/restart-chaos/wake-evidence.test.ts +++ b/tests/restart-chaos/wake-evidence.test.ts @@ -152,7 +152,8 @@ describe.skipIf(!support.supported)("wake evidence against real PostgreSQL", () owner: { pid: 4242, processSpace: "space-wake" }, providerTurnStarted: false }); const slots = JSON.parse(await readFile(path.join(waveDir, "wake-evidence", "agent-slots.json"), "utf8")); expect(slots.agents).toEqual(expect.arrayContaining([{ agentId, maxConcurrentRuns: 3 }, { agentId: wideAgentId, maxConcurrentRuns: 50 }])); - expect(slots.runs).toEqual([{ runId, agentId, startedAt: 1_800_000_000_000, finishedAt: 1_800_000_060_000 }]); + expect(slots.runs).toEqual([{ runId, agentId, createdAt: evidence.runs[0].createdAt, invocationSource: "on_demand", retryOfRunId: null, startedAt: 1_800_000_000_000, finishedAt: 1_800_000_060_000 }]); + expect(slots.runs[0].createdAt).toEqual(expect.any(Number)); expect(evidence.wakes[0]).toMatchObject({ id: wakeId, reason: "retry", source: "on_demand", triggerDetail: "manual", requestedByActorType: "user", requestedByActorId: "[REDACTED]", idempotencyKey: "idem-1", runId }); expect(evidence.wakes[0].servingAtCreate.map((server: { pid: number }) => server.pid)).toEqual([4242]); diff --git a/tests/restart-chaos/wave-grade-inputs.ts b/tests/restart-chaos/wave-grade-inputs.ts index a2d7295e557..ced6af25707 100644 --- a/tests/restart-chaos/wave-grade-inputs.ts +++ b/tests/restart-chaos/wave-grade-inputs.ts @@ -1,6 +1,6 @@ import { readFile } from "node:fs/promises"; import path from "node:path"; -import { AGENT_SLOTS_FILE, slotFreeFrom, type AgentSlots } from "./agent-slots.js"; +import { AGENT_SLOTS_FILE, queueWaitFairnessFrom, slotFreeFrom, type AgentSlots, type QueueWaitFairness } from "./agent-slots.js"; import { withChildWaits } from "./child-wait.js"; import type { WakeEvidenceFile } from "./staging-mix-retry.js"; import type { StagingMixOptions } from "./staging-mix.js"; @@ -8,7 +8,7 @@ import type { TaskRecord } from "./types.js"; import { declaredQueueBoundMs, universalBounds, type SlotFree, type UniversalBounds } from "./universal.js"; export interface WaveGradeInputs { - queue: { bounds: UniversalBounds; slotFree: SlotFree }; + queue: { bounds: UniversalBounds; slotFree: SlotFree; queueWait: QueueWaitFairness }; slotEvidence: boolean; prepare(taskId: string, record: TaskRecord): Promise; } @@ -21,7 +21,7 @@ export async function loadWaveGradeInputs(waveDir: string, mix: Pick withChildWaits(record, await readJson(path.join(waveDir, "wake-evidence", `${taskId}.json`))), };