diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index ba83df15fd7..636e56f5a27 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -406,14 +406,28 @@ part of U3 is checked only in waves run with this harness. - R1 allows the workload budget plus outage time. When a Retry was needed R1 is `n/a` and R3 decides; a task that rests unfinished with no Retry fails both. A task that never rested within its deadline fails R1 as `no_rest_within_budget`, - whatever its last status. R1 also allows the first run's initial queue wait - (minus outage time inside it) when `wake-evidence/agent-slots.json` proves it - fair: no newer fresh assignment run of that agent started during the wait - (newer follow-ups and retries rank higher by design), and the agent had no - idle slot for more than 2 s outside a server outage. A provably unfair wait - fails as `unfair_queue_wait`; one that cannot be proven is not credited, and - the R1 detail says why. - Waits of later runs (recovery, restarts, retries) are never credited. + whatever its last status. R1 also allows fair queue waits of runs that new + input started, chosen by these rules in order: + 1. If any run on the task has no `createdAt`, nothing is credited. + 2. The first run (earliest `createdAt`) is a candidate unless it is a retry. + 3. For each harness follow-up comment: without a send time, nothing is + credited for it. Otherwise the candidate is the earliest run that carries + the comment, was created at or after the send time, and was not cancelled + before it executed. + 4. That run counts only if it is not a retry, the previous executed run + succeeded and finished before it was created, and + `wake-evidence/.json` shows a comment wake reason and no continued + run (no wake evidence: nothing is credited). + 5. Each candidate's wait, from `createdAt` to `startedAt` inside the task's + own lifetime and minus outage time, is credited only when + `wake-evidence/agent-slots.json` proves it fair: no newer fresh assignment + run of that agent started during the wait (newer follow-ups and retries + rank higher by design), and the agent had no idle slot for more than 2 s + outside a server outage. A provably unfair wait fails as + `unfair_queue_wait`; one that cannot be proven is not credited, and the R1 + detail says why. + The observer holds the deadline while a candidate run is queued (up to one + more budget) and extends it by that run's wait once it starts. - Speed: an aggregate p50/p95 row can breach only with at least 5 samples; a smaller row is reported as `insufficient sample (n < 5)`. Every cold or warm run over its p95 budget, and every restart recovery whose Paperclip share exceeds diff --git a/tests/restart-chaos/grader.test.ts b/tests/restart-chaos/grader.test.ts index b1b63e77195..169332956c8 100644 --- a/tests/restart-chaos/grader.test.ts +++ b/tests/restart-chaos/grader.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { QUEUE_HANDOFF_TOLERANCE_MS, queueWaitFairnessFrom, type AgentSlots } from "./agent-slots.js"; import { crossingFaults, gradeR1, gradeR2, gradeR3, gradeR4, gradeR5, gradeTask, STALL_LIMIT_MS } from "./grader.js"; import { isResting } from "./observer.js"; +import { runWakesFrom } from "./wave-grade-inputs.js"; import type { FaultReceipt, OracleArtifacts, RunSample, TaskRecord, TaskSample } from "./types.js"; import { BENCH_EXPECTED, parseBenchRecord, r1TestChecksSlug, stallWakeChecks, workloadDefinitions } from "./workloads.js"; @@ -315,6 +316,185 @@ describe("R1 fair queue wait credit", () => { }); }); +describe("R1 follow-up queue wait credit", () => { + const BUDGET = 2_010_000; + const WAIT = 1_558_000; + type SlotRun = AgentSlots["runs"][number]; + const slotRun = (runId: string, created: number, started: number, finished: number | null, extra: Partial = {}): SlotRun => + ({ runId, agentId: "agent-oc", createdAt: T0 + created, invocationSource: "assignment", retryOfRunId: null, startedAt: T0 + started, + finishedAt: finished === null ? null : T0 + finished, ...extra }); + const shape = (firstWait = 0) => { + const firstStart = 50 + firstWait; + const firstEnd = firstStart + 666_000; + const queuedAt = firstEnd + 50; + const startedAt = queuedAt + WAIT; + return { firstStart, firstEnd, queuedAt, startedAt }; + }; + const agent = (firstWait = 0, followUp: Partial = {}): SlotRun[] => { + const { firstStart, firstEnd, queuedAt, startedAt } = shape(firstWait); + return [ + slotRun("r1", 50, firstStart, firstEnd), + slotRun("r2", queuedAt, startedAt, null, { invocationSource: "automation", ...followUp }), + slotRun("o1", -900_000, -800_000, firstStart), + slotRun("o2", -900_000, -700_000, 1_400_000 + firstWait), + slotRun("o3", -900_000, -600_000, startedAt), + slotRun("o4", -900_000, -500_000, 600_000 + firstWait), + slotRun("o5", 10, 600_000 + firstWait, 2_400_000 + firstWait), + slotRun("o6", 20, firstEnd, 2_500_000 + firstWait), + slotRun("o7", 30, 1_400_000 + firstWait, 2_600_000 + firstWait), + ]; + }; + const slots = (runs: SlotRun[]): AgentSlots => ({ agents: [{ agentId: "agent-oc", maxConcurrentRuns: 4 }], runs }); + const followUpTask = (tookMs: number, firstWait = 0, follow: Partial = {}, first: Partial = {}, + extra: Partial = {}, others: RunSample[] = []) => { + const { firstStart, firstEnd, queuedAt, startedAt } = shape(firstWait); + const r1 = run("r1", "succeeded", 50, firstEnd, { startedAt: iso(firstStart), ...first }); + const r2 = (status: string, started: number | null, finished: number | null) => + run("r2", status, queuedAt, finished, { startedAt: started === null ? null : iso(started), wakeCommentIds: ["c1"], ...follow }); + return record([ + sample(queuedAt + 5_000, "in_progress", [r1, r2("queued", null, null), ...others]), + sample(startedAt + 5_000, "in_progress", [r1, r2("running", startedAt, null), ...others]), + sample(tookMs, "done", [r1, r2("succeeded", startedAt, tookMs - 1_000), ...others]), + ], { restAt: T0 + tookMs, faultIds: [], plan: { ...record([]).plan, workload: "SM" }, + stagingMix: { followUp: { at: T0 + 200_000, commentId: "c1", duringRun: true, inOverlap: false, error: null } }, + runWakes: { r1: { wakeReason: "issue_assigned", continuesRunIds: [] }, r2: { wakeReason: "issue_commented", continuesRunIds: [] } }, ...extra }); + }; + const t007 = () => followUpTask(2_310_000); + + it("credits a fair 1,558 s follow-up queue wait behind the agent's older assignments", () => { + expect(gradeR1(t007(), BUDGET, [])).toEqual({ verdict: "fail", detail: "done_too_late 2310s > 2010s (queue wait not credited: no agent slot evidence for r2)" }); + expect(gradeR1(t007(), BUDGET, [], queueWaitFairnessFrom(slots(agent())))) + .toEqual({ verdict: "pass", detail: "done after 2310s (allowed 3568s incl. 1558s fair queue wait)" }); + }); + + it("fails as unfair when a newer fresh assignment started ahead of the follow-up", () => { + const runs = agent().map(entry => entry.runId === "o7" ? { ...entry, createdAt: T0 + 700_000 } : entry); + const graded = gradeR1(t007(), BUDGET, [], queueWaitFairnessFrom(slots(runs))); + expect(graded.verdict).toBe("fail"); + expect(graded.detail).toMatch(/^unfair_queue_wait assignment run o7 created .* ahead of r2; done 2310s > 2010s without queue credit$/); + const task = gradeTask(t007(), [], BUDGET, oracle(), undefined, { queueWait: queueWaitFairnessFrom(slots(runs)) }); + expect(task.signature).toMatch(/^R1:unfair_queue_wait/); + }); + + it("fails as unfair when a slot sat idle while the follow-up was queued", () => { + const runs = agent().map(entry => entry.runId === "o7" ? { ...entry, startedAt: T0 + 1_431_000 } : entry); + const graded = gradeR1(t007(), BUDGET, [], queueWaitFairnessFrom(slots(runs))); + expect(graded.verdict).toBe("fail"); + expect(graded.detail).toContain(`idle slot from ${iso(1_400_000)} to ${iso(1_431_000)} while r2 was queued`); + }); + + it("does not credit the wait of a retry or a recovery run", () => { + const fairness = queueWaitFairnessFrom(slots(agent())); + const retry = gradeR1(followUpTask(2_310_000, 0, { retryOfRunId: "r1" }), BUDGET, [], fairness); + expect(retry).toEqual({ verdict: "fail", detail: "done_too_late 2310s > 2010s" }); + const afterInterruption = gradeR1(followUpTask(2_310_000, 0, {}, { status: "failed", errorCode: "process_lost" }), BUDGET, [], fairness); + expect(afterInterruption).toEqual({ verdict: "fail", detail: "done_too_late 2310s > 2010s" }); + const continuation = gradeR1(followUpTask(2_310_000, 0, { wakeCommentIds: [] }), BUDGET, [], fairness); + expect(continuation).toEqual({ verdict: "fail", detail: "done_too_late 2310s > 2010s" }); + }); + + it("does not credit a follow-up run created while the previous run was still running", () => { + const { queuedAt } = shape(); + const overlapping = followUpTask(2_310_000, 0, {}, { finishedAt: iso(queuedAt + 10_000) }); + expect(gradeR1(overlapping, BUDGET, [], queueWaitFairnessFrom(slots(agent())))).toEqual({ verdict: "fail", detail: "done_too_late 2310s > 2010s" }); + }); + + it("credits a fair wait of a run woken by an after-end follow-up, but not after a recovery", () => { + const { firstEnd } = shape(); + const afterEnd = { stagingMix: { afterEndFollowUps: [{ afterRunEndAt: T0 + firstEnd, at: T0 + firstEnd + 10, delayMs: 10, commentId: "c1" }] } }; + const fairness = queueWaitFairnessFrom(slots(agent())); + expect(gradeR1(followUpTask(2_310_000, 0, {}, {}, afterEnd), BUDGET, [], fairness)) + .toEqual({ verdict: "pass", detail: "done after 2310s (allowed 3568s incl. 1558s fair queue wait)" }); + expect(gradeR1(followUpTask(2_310_000, 0, {}, { status: "failed", errorCode: "process_lost" }, afterEnd), BUDGET, [], fairness)) + .toEqual({ verdict: "fail", detail: "done_too_late 2310s > 2010s" }); + }); + + it("refuses follow-up credit when an earlier run has no createdAt", () => { + const undated = run("rx", "failed", 100_000, 600_000, { createdAt: null, errorCode: "process_lost" }); + const graded = gradeR1(followUpTask(2_310_000, 0, {}, {}, {}, [undated]), BUDGET, [], queueWaitFairnessFrom(slots(agent()))); + expect(graded).toEqual({ verdict: "fail", + detail: "done_too_late 2310s > 2010s (queue wait not credited: no createdAt for rx, so runs cannot be ordered)" }); + }); + + it("does not credit a recovery run that carries the comment after a success", () => { + const fairness = queueWaitFairnessFrom(slots(agent())); + for (const r2 of [{ wakeReason: "issue_continuation_needed", continuesRunIds: [] }, { wakeReason: "issue_commented", continuesRunIds: ["r1"] }]) { + const recovery = followUpTask(2_310_000, 0, {}, {}, { runWakes: { r2 } }); + expect(gradeR1(recovery, BUDGET, [], fairness)).toEqual({ verdict: "fail", detail: "done_too_late 2310s > 2010s" }); + } + }); + + it("reads each run's wake reason and continued runs from the wake evidence", () => { + const run = (runId: string, wakeupRequestId: string | null, contextSnapshot: unknown) => + ({ runId, createdAt: 1, invocationSource: "automation", triggerDetail: "system", wakeupRequestId, contextSnapshot }); + expect(runWakesFrom({ + runs: [run("r2", "w2", { wakeReason: "issue_commented" }), run("r3", "w3", { executionContinuation: { interruptedRunId: "r2" } })], + wakes: [{ id: "w2", reason: "issue_execution_promoted", source: "automation", triggerDetail: "system", requestedByActorType: "user", requestedByActorId: "u" }, + { id: "w3", reason: "issue_continuation_needed", source: "automation", triggerDetail: "system", requestedByActorType: null, requestedByActorId: null }], + })).toEqual({ r2: { wakeReason: "issue_commented", continuesRunIds: [] }, r3: { wakeReason: "issue_continuation_needed", continuesRunIds: ["r2"] } }); + }); + + it("does not credit a run created before its follow-up comment was posted", () => { + const { queuedAt } = shape(); + const fairness = queueWaitFairnessFrom(slots(agent())); + const early = (at: number | undefined) => followUpTask(2_310_000, 0, {}, {}, + { stagingMix: { followUp: { at: at as number, commentId: "c1", duringRun: true, inOverlap: false, error: null } } }); + expect(gradeR1(early(T0 + queuedAt + 200_000), BUDGET, [], fairness)).toEqual({ verdict: "fail", detail: "done_too_late 2310s > 2010s" }); + expect(gradeR1(early(undefined), BUDGET, [], fairness)) + .toEqual({ verdict: "fail", detail: "done_too_late 2310s > 2010s (queue wait not credited: no send time for follow-up comment c1)" }); + }); + + it("credits a run created after the comment request was sent but before it returned", () => { + const { queuedAt } = shape(); + const followUp = { at: T0 + queuedAt - 1_000, sentAt: T0 + queuedAt - 500, postedAt: T0 + queuedAt + 400, commentId: "c1", error: null }; + expect(gradeR1(followUpTask(2_310_000, 0, {}, {}, { stagingMix: {}, followUp }), BUDGET, [], queueWaitFairnessFrom(slots(agent())))) + .toEqual({ verdict: "pass", detail: "done after 2310s (allowed 3568s incl. 1558s fair queue wait)" }); + }); + + it("credits nothing when the earliest run has no createdAt", () => { + const undated = run("r0", "failed", -10_000, 40, { createdAt: null, errorCode: "process_lost" }); + const graded = gradeR1(followUpTask(2_610_000, 300_000, {}, {}, {}, [undated]), BUDGET, [], queueWaitFairnessFrom(slots(agent(300_000)))); + expect(graded).toEqual({ verdict: "fail", detail: "done_too_late 2610s > 2010s (queue wait not credited: no createdAt for r0, so runs cannot be ordered)" }); + }); + + it("credits the first executed carrier, never a run cancelled before it executed", () => { + const { queuedAt } = shape(); + const fairness = queueWaitFairnessFrom(slots(agent())); + const cancelled = (errorCode: string, started: boolean) => run("rc", "cancelled", queuedAt - 20, queuedAt - 10, + { wakeCommentIds: ["c1"], errorCode, startedAt: started ? iso(queuedAt - 15) : null }); + const followUp = { at: T0 + queuedAt - 1_000, sentAt: T0 + queuedAt - 500, commentId: "c1", error: null }; + const runWakes = { r2: { wakeReason: "issue_commented", continuesRunIds: [] }, rc: { wakeReason: "issue_commented", continuesRunIds: [] } }; + for (const [errorCode, started] of [["continuation_task_ownership_changed", true], ["cancelled", false]] as const) { + const task = followUpTask(2_310_000, 0, {}, {}, { stagingMix: {}, followUp, runWakes }, [cancelled(errorCode, started)]); + expect(gradeR1(task, BUDGET, [], fairness)).toEqual({ verdict: "pass", detail: "done after 2310s (allowed 3568s incl. 1558s fair queue wait)" }); + } + }); + + it("refuses follow-up credit without wake evidence", () => { + const graded = gradeR1(followUpTask(2_310_000, 0, {}, {}, { runWakes: undefined }), BUDGET, [], queueWaitFairnessFrom(slots(agent()))); + expect(graded).toEqual({ verdict: "fail", detail: "done_too_late 2310s > 2010s (queue wait not credited: no wake evidence for follow-up run r2)" }); + }); + + it("credits only the part of a wait inside the task's own lifetime", () => { + const graded = gradeR1(followUpTask(2_310_000, 0, {}, {}, { restAt: T0 + 2_100_000 }), BUDGET, [], queueWaitFairnessFrom(slots(agent()))); + expect(graded).toEqual({ verdict: "pass", detail: "done after 2100s (allowed 3444s incl. 1434s fair queue wait)" }); + }); + + it("does not credit outage time inside the follow-up's wait", () => { + const { queuedAt } = shape(); + const outage: FaultReceipt = { ...fault, startedAt: T0 + queuedAt + 100_000, readyAt: T0 + queuedAt + 200_000 }; + expect(gradeR1(followUpTask(2_500_000), BUDGET, [outage], queueWaitFairnessFrom(slots(agent())))) + .toEqual({ verdict: "pass", detail: "done after 2500s (allowed 3568s incl. 1458s fair queue wait)" }); + }); + + it("sums the fair waits of the first run and a follow-up", () => { + const graded = gradeR1(followUpTask(2_610_000, 300_000), BUDGET, [], queueWaitFairnessFrom(slots(agent(300_000)))); + expect(graded).toEqual({ verdict: "pass", detail: "done after 2610s (allowed 3868s incl. 1858s fair queue wait)" }); + const tooLate = gradeR1(followUpTask(3_900_000, 300_000), BUDGET, [], queueWaitFairnessFrom(slots(agent(300_000)))); + expect(tooLate).toEqual({ verdict: "fail", detail: "done_too_late 3900s > 3868s incl. 1858s fair queue wait" }); + }); +}); + describe("T-oracles", () => { it("grades M1 delegation: two done subtasks, part 1 delegated, and a summary of both", () => { const agents = { self: { id: "self", name: "Chaos codex" }, other: { id: "other", name: "Chaos claude" } }; diff --git a/tests/restart-chaos/grader.ts b/tests/restart-chaos/grader.ts index e25252954a9..2e831c90596 100644 --- a/tests/restart-chaos/grader.ts +++ b/tests/restart-chaos/grader.ts @@ -63,25 +63,61 @@ export function expectedFinalStatus(record: TaskRecord): string { return record.plan.workload === "G1" && g1Decision(record.plan.nonce) === "reject" ? "cancelled" : "done"; } -export function firstRun(record: TaskRecord): NonNullable[number] | null { - const latest = new Map[number]>(); +type TaskRun = NonNullable[number]; +export type CreditedRun = { run: TaskRun; kind: "first run" | "follow-up run" }; + +function boardComments(record: TaskRecord): Map { + const posted = (at: number | undefined) => typeof at === "number" && Number.isFinite(at) ? at : null; + const entries: Array<[string | null | undefined, number | null]> = [ + [record.followUp?.commentId, posted(record.followUp?.sentAt ?? record.followUp?.at)], + [record.stagingMix?.followUp?.commentId, posted(record.stagingMix?.followUp?.at)], + ...(record.stagingMix?.afterEndFollowUps ?? []).map((entry): [string, number | null] => [entry.commentId, posted(entry.at)]), + ]; + return new Map(entries.filter((entry): entry is [string, number | null] => typeof entry[0] === "string" && entry[0].length > 0)); +} + +const COMMENT_WAKE_REASONS = new Set(["issue_commented", "issue_comment_mentioned", "issue_reopened_via_comment"]); + +function runSelection(record: TaskRecord, requireWakeEvidence: boolean): { credited: CreditedRun[]; refusals: string[] } { + const latest = new Map(); for (const sample of record.samples) for (const run of sample.runs ?? []) latest.set(run.id, run); - const first = [...latest.values()].filter(run => Number.isFinite(ms(run.createdAt))) - .sort((a, b) => ms(a.createdAt) - ms(b.createdAt))[0]; - return first && !first.retryOfRunId ? first : null; + const undated = [...latest.values()].filter(run => !Number.isFinite(ms(run.createdAt))).map(run => run.id); + if (undated.length) return { credited: [], refusals: [`no createdAt for ${undated.join(",")}, so runs cannot be ordered`] }; + const runs = [...latest.values()].sort((a, b) => ms(a.createdAt) - ms(b.createdAt)); + const credited: CreditedRun[] = runs[0] && !runs[0].retryOfRunId ? [{ run: runs[0], kind: "first run" }] : []; + const refusals: string[] = []; + const unexecuted = (run: TaskRun) => run.status === "cancelled" && (!run.startedAt || PRE_DISPATCH_CANCELLATIONS.has(run.errorCode ?? "")); + for (const [commentId, sentAt] of boardComments(record)) { + const carriers = runs.filter(run => (run.wakeCommentIds ?? []).includes(commentId)); + if (!carriers.length) continue; + if (sentAt === null) { refusals.push(`no send time for follow-up comment ${commentId}`); continue; } + const run = carriers.find(candidate => ms(candidate.createdAt) >= sentAt && !unexecuted(candidate)); + const previous = run && runs.filter(other => ms(other.createdAt) < ms(run.createdAt) && !unexecuted(other)).at(-1); + if (!run || !previous || run.retryOfRunId || previous.status !== "succeeded" || !(ms(previous.finishedAt) <= ms(run.createdAt))) continue; + if (credited.some(entry => entry.run.id === run.id)) continue; + const wake = record.runWakes?.[run.id]; + if (requireWakeEvidence && !wake) { refusals.push(`no wake evidence for follow-up run ${run.id}`); continue; } + if (wake && (wake.continuesRunIds.length || !COMMENT_WAKE_REASONS.has(wake.wakeReason ?? ""))) continue; + credited.push({ run, kind: "follow-up run" }); + } + return { credited, refusals }; } -export function firstRunQueuedMs(record: TaskRecord, now: number): number { - const first = firstRun(record); - return first && first.status === "queued" && !first.startedAt ? Math.max(0, now - ms(first.createdAt)) : 0; +export function creditedRuns(record: TaskRecord): CreditedRun[] { + return runSelection(record, true).credited; } -export function firstRunQueueWait(record: TaskRecord): { runId: string; createdAt: number; startedAt: number } | null { - const first = firstRun(record); - if (!first || !Number.isFinite(ms(first.startedAt))) return null; - const createdAt = ms(first.createdAt); - const startedAt = ms(first.startedAt); - return startedAt > createdAt ? { runId: first.id, createdAt, startedAt } : null; +function queueWaitsOf(credited: readonly CreditedRun[]): Array<{ runId: string; createdAt: number; startedAt: number }> { + return credited.map(({ run }) => ({ runId: run.id, createdAt: ms(run.createdAt), startedAt: ms(run.startedAt) })) + .filter(wait => Number.isFinite(wait.startedAt) && wait.startedAt > wait.createdAt); +} + +export function creditedRunQueuedMs(record: TaskRecord, now: number): number { + return Math.max(0, ...runSelection(record, false).credited.filter(({ run }) => run.status === "queued" && !run.startedAt).map(({ run }) => now - ms(run.createdAt))); +} + +export function creditedQueueWaits(record: TaskRecord): Array<{ runId: string; createdAt: number; startedAt: number }> { + return queueWaitsOf(runSelection(record, false).credited); } function faultOverlapMs(faults: readonly FaultReceipt[], from: number, to: number): number { @@ -105,16 +141,26 @@ export function gradeR1(record: TaskRecord, budgetMs: number, faults: readonly F const took = (record.restAt ?? Number.POSITIVE_INFINITY) - record.createdAt!; const seconds = (value: number) => Math.round(value / 1000); if (took <= base) return { verdict: "pass", detail: `${expected} after ${seconds(took)}s (allowed ${seconds(base)}s)` }; - const first = firstRun(record); - const wait = firstRunQueueWait(record); - const fairness: QueueFairness | null = wait ? queueWait(wait.runId, wait.createdAt, wait.startedAt, faults.map(faultWindow)) - : first && !first.startedAt ? { verdict: "unproven", reason: `first run ${first.id} never started` } : null; - if (fairness?.verdict === "unfair") { - return { verdict: "fail", detail: `unfair_queue_wait ${fairness.reason}; done ${seconds(took)}s > ${seconds(base)}s without queue credit` }; + const { credited, refusals } = runSelection(record, true); + const waits = new Map(queueWaitsOf(credited).map(wait => [wait.runId, wait])); + const proofs = credited.flatMap(({ run, kind }): Array<{ fairness: QueueFairness; credit: number }> => { + const wait = waits.get(run.id); + if (wait) { + const from = Math.max(wait.createdAt, record.createdAt!); + const to = Math.min(wait.startedAt, record.restAt ?? wait.startedAt); + return [{ fairness: queueWait(wait.runId, wait.createdAt, wait.startedAt, faults.map(faultWindow)), + credit: Math.max(0, to - from - faultOverlapMs(faults, from, to)) }]; + } + return run.startedAt ? [] : [{ fairness: { verdict: "unproven", reason: `${kind} ${run.id} never started` }, credit: 0 }]; + }); + const unfair = proofs.map(proof => proof.fairness).find(fairness => fairness.verdict === "unfair"); + if (unfair && unfair.verdict === "unfair") { + return { verdict: "fail", detail: `unfair_queue_wait ${unfair.reason}; done ${seconds(took)}s > ${seconds(base)}s without queue credit` }; } - const credit = wait && fairness?.verdict === "fair" ? Math.max(0, wait.startedAt - wait.createdAt - faultOverlapMs(faults, wait.createdAt, wait.startedAt)) : 0; + const credit = proofs.reduce((total, proof) => total + (proof.fairness.verdict === "fair" ? proof.credit : 0), 0); + const unproven = [...proofs.flatMap(({ fairness }) => fairness.verdict === "unproven" ? [fairness.reason] : []), ...refusals]; const allowed = base + credit; - const note = credit ? ` incl. ${seconds(credit)}s fair queue wait` : fairness?.verdict === "unproven" ? ` (queue wait not credited: ${fairness.reason})` : ""; + const note = `${credit ? ` incl. ${seconds(credit)}s fair queue wait` : ""}${unproven.length ? ` (queue wait not credited: ${unproven.join("; ")})` : ""}`; return took <= allowed ? { verdict: "pass", detail: `${expected} after ${seconds(took)}s (allowed ${seconds(allowed)}s${note})` } : { verdict: "fail", detail: `done_too_late ${seconds(took)}s > ${seconds(allowed)}s${note}` }; diff --git a/tests/restart-chaos/observer.test.ts b/tests/restart-chaos/observer.test.ts index a4364cdb812..790c875236a 100644 --- a/tests/restart-chaos/observer.test.ts +++ b/tests/restart-chaos/observer.test.ts @@ -142,6 +142,48 @@ describe("TaskObserver", () => { expect(observer.record.timedOut).toBe(true); }); + it("waits while a follow-up run is queued and extends the deadline by its queue wait once it starts", async () => { + let queuedAt = 0; + let startedAt: number | null = null; + const first = () => ({ runId: "r1", status: "succeeded", createdAt: new Date(queuedAt - 60_000).toISOString(), + startedAt: new Date(queuedAt - 60_000).toISOString(), finishedAt: new Date(queuedAt - 1).toISOString() }); + const { api } = fakeApi({ "/runs": () => [first(), { runId: "r2", status: startedAt === null ? "queued" : "running", wakeCommentIds: ["comment-1"], + createdAt: new Date(queuedAt).toISOString(), startedAt: startedAt === null ? null : new Date(startedAt).toISOString() }] }, "in_progress"); + const observer = new TaskObserver({ ...plan, workload: "A1", followUpAfterMs: null }, api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + const deadline = observer.record.deadlineAt!; + const budget = deadline - start; + queuedAt = start + 60_000; + observer.record.followUp = { at: start + 30_000, commentId: "comment-1", error: null }; + await observer.poll(deadline + 1_000); + expect(observer.record.timedOut).toBe(false); + expect(observer.record.deadlineAt).toBe(deadline); + startedAt = deadline + 120_000; + await observer.poll(deadline + 121_000); + expect(observer.record.timedOut).toBe(false); + expect(observer.record.deadlineAt).toBe(deadline + startedAt - queuedAt); + await observer.poll(deadline + 122_000); + expect(observer.record.deadlineAt).toBe(deadline + startedAt - queuedAt); + expect(startedAt - queuedAt).toBeLessThan(2 * budget); + }); + + it("does not hold the deadline for a queued run that a recovery created", async () => { + let queuedAt = 0; + const { api } = fakeApi({ "/runs": () => [ + { runId: "r1", status: "failed", createdAt: new Date(queuedAt - 60_000).toISOString(), startedAt: new Date(queuedAt - 60_000).toISOString(), + finishedAt: new Date(queuedAt - 1).toISOString() }, + { runId: "r2", status: "queued", wakeCommentIds: ["comment-1"], createdAt: new Date(queuedAt).toISOString(), startedAt: null }] }, "in_progress"); + const observer = new TaskObserver({ ...plan, workload: "A1", followUpAfterMs: null }, api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + const deadline = observer.record.deadlineAt!; + queuedAt = start + 60_000; + observer.record.followUp = { at: start + 30_000, commentId: "comment-1", error: null }; + await observer.poll(deadline + 1_000); + expect(observer.record.timedOut).toBe(true); + }); + it("keeps watching unexplained idle work until the stall limit, then records the stall", async () => { const { api } = fakeApi({}, "in_progress"); const observer = new TaskObserver({ ...plan, workload: "A1", followUpAfterMs: null }, api, lab, new Timeline(null), ready); diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index 8c43a078d25..909a13da5df 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -6,7 +6,7 @@ import os from "node:os"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { ChaosApiError, type ChaosApi } from "./api.js"; -import { ACTIVE_RUN_STATUSES, firstRunQueuedMs, firstRunQueueWait, isStalledSample, STALL_LIMIT_MS } from "./grader.js"; +import { ACTIVE_RUN_STATUSES, creditedQueueWaits, creditedRunQueuedMs, isStalledSample, STALL_LIMIT_MS } from "./grader.js"; import type { ChaosLab } from "./seed.js"; import type { Timeline } from "./timeline.js"; import type { @@ -218,7 +218,7 @@ export class TaskObserver { private stallSince: number | null = null; /** First sample of the current stretch with the server ready (H1 holds the pause across it). */ private readySince: number | null = null; - private queueWaitExtended = false; + private readonly queueWaitsExtended = new Set(); private readonly faultSchedule: ScheduledFault[]; private readonly pendingFaults: ScheduledFault[]; done = false; @@ -492,19 +492,19 @@ export class TaskObserver { /** Outages extend the deadline: R1 allows budget plus outage time. */ extendDeadline(ms: number) { if (this.record.deadlineAt !== null) this.record.deadlineAt += ms; } - private extendForFirstQueueWait() { - if (this.queueWaitExtended) return; - const wait = firstRunQueueWait(this.record); - if (!wait) return; - this.queueWaitExtended = true; - this.extendDeadline(wait.startedAt - wait.createdAt); + private extendForQueueWaits() { + for (const wait of creditedQueueWaits(this.record)) { + if (this.queueWaitsExtended.has(wait.runId)) continue; + this.queueWaitsExtended.add(wait.runId); + this.extendDeadline(wait.startedAt - wait.createdAt); + } } async poll(now = Date.now()): Promise { if (this.done || !this.record.issueId) return; const sample = await this.sample(now); this.record.samples.push(sample); - this.extendForFirstQueueWait(); + this.extendForQueueWaits(); await this.captureRemoteFilesAtDone(sample); this.readySince = sample.reachable && sample.serverReady ? (this.readySince ?? now) : null; await this.observeBenchProgress(); @@ -552,7 +552,7 @@ export class TaskObserver { if (this.done) return; } } - const queuedMs = this.queueWaitExtended ? 0 : Math.min(firstRunQueuedMs(this.record, now), this.definition.budgetMs); + const queuedMs = Math.min(creditedRunQueuedMs(this.record, now), this.definition.budgetMs); if (this.record.deadlineAt !== null && now > this.record.deadlineAt + queuedMs && !this.supervisor.faultInProgress) { if (this.restCandidateSince !== null && this.restCandidateSince <= this.record.deadlineAt && now - this.restCandidateSince < REST_CONFIRM_MS) return; diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index 1a90635ab39..37e6fb96391 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -440,6 +440,8 @@ export interface RestSweep { error: string | null; } +export interface RunWake { wakeReason: string | null; continuesRunIds: string[] } + export interface TaskRecord { plan: TaskPlan; issueId: string | null; @@ -463,6 +465,7 @@ export interface TaskRecord { restSnapshotError?: string; priorRetries?: RetryAttempt[]; childStalls?: Array<{ childIssueId: string; recoveryActionId: string | null }>; + runWakes?: Record; childRetries?: Array; workspaceMovedTo?: string; /** Set when the pre-fault snapshot could not read the workspace; R2 then fails visibly. */ diff --git a/tests/restart-chaos/wave-grade-inputs.ts b/tests/restart-chaos/wave-grade-inputs.ts index ced6af25707..999462c26c8 100644 --- a/tests/restart-chaos/wave-grade-inputs.ts +++ b/tests/restart-chaos/wave-grade-inputs.ts @@ -2,9 +2,10 @@ import { readFile } from "node:fs/promises"; import path from "node:path"; import { AGENT_SLOTS_FILE, queueWaitFairnessFrom, slotFreeFrom, type AgentSlots, type QueueWaitFairness } from "./agent-slots.js"; import { withChildWaits } from "./child-wait.js"; +import { continuedRunIds } from "./follow-up-delivery.js"; import type { WakeEvidenceFile } from "./staging-mix-retry.js"; import type { StagingMixOptions } from "./staging-mix.js"; -import type { TaskRecord } from "./types.js"; +import type { RunWake, TaskRecord } from "./types.js"; import { declaredQueueBoundMs, universalBounds, type SlotFree, type UniversalBounds } from "./universal.js"; export interface WaveGradeInputs { @@ -23,6 +24,21 @@ export async function loadWaveGradeInputs(waveDir: string, mix: Pick withChildWaits(record, await readJson(path.join(waveDir, "wake-evidence", `${taskId}.json`))), + prepare: async (taskId, record) => { + const evidence = await readJson(path.join(waveDir, "wake-evidence", `${taskId}.json`)); + const prepared = withChildWaits(record, evidence); + return evidence ? { ...prepared, runWakes: runWakesFrom(evidence) } : prepared; + }, }; } + +export function runWakesFrom(evidence: WakeEvidenceFile): Record { + const wakes = new Map((evidence.wakes ?? []).map(wake => [wake.id, wake])); + return Object.fromEntries((evidence.runs ?? []).flatMap(run => { + if (!run.runId) return []; + const wake = run.wakeupRequestId ? wakes.get(run.wakeupRequestId) : undefined; + const context = run.contextSnapshot && typeof run.contextSnapshot === "object" ? run.contextSnapshot as Record : {}; + const wakeReason = typeof context.wakeReason === "string" ? context.wakeReason : wake?.reason ?? null; + return [[run.runId, { wakeReason, continuesRunIds: continuedRunIds(run, wake) }]]; + })); +}