diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index b2f2a52ccbf..820eddc7225 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -557,6 +557,52 @@ node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona --daytona-warm --daytona-cpu 2 --adapters opencode,codex,claude --release-swap-short --concurrency 12 --tasks 200 --seed 7110 ``` +With `--workload staging-mix`, the swaps become the mix's fault slots (`--mix-fault-spacing-minutes` and `--mix-minutes` +come from the swap count and spacing), so the staging-mix tasks, follow-ups and hands-off results run under the swap +driver's probes and grading. Probe agents finish their verification while the first tasks run; the first swap waits for them. +`--server-from ` also runs on Daytona: every server comes from that checkout while the sandboxes keep +`--daytona-image`'s Runner and provider pack, as a deploy newer or older than its sandbox image does. + +### Staging twin cells + +Two local servers, volume-free, real Daytona sandboxes, two overlap swaps. `$IMG` must match the checkout's Runner +(`--server-from`'s checkout when given). Run each through the wave queue (`~/.cache/paperclip/chaos-wave-acquire.sh`). + +```sh +set -a; source ~/.secrets; set +a +# Twin cell, variant G (graceful swaps): 6 tasks, 6 runs, 4-minute turns, 2 swaps 4 minutes apart, 120 s overlap. 18-23 minutes. +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona --daytona-image $IMG --daytona-warm --daytona-cpu 1 --storage s3 \ + --workload staging-mix --mix-tasks 6 --mix-runs 6 --mix-queue 0 --mix-ratio opencode=34,codex=33,claude=33 --mix-turn-minutes 4 \ + --mix-hold-minutes 2 --mix-arrival-cutoff-minutes 0 --release-swap --release-swap-count 2 --release-swap-spacing-minutes 4 \ + --release-swap-overlap-seconds 120 --release-swap-settle-minutes 1 --seed 7105 +# Variant G2: the same command; the outgoing server is SIGKILLed 120 s into each overlap. +RESTART_CHAOS_K15_KILL_OLD_AFTER_SECONDS=120 node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts ... +# Variant G3: G2, and the incoming server is also frozen (SIGSTOP) for 150 s while the outgoing one stops, so it reaches the Runners late. +RESTART_CHAOS_K15_KILL_OLD_AFTER_SECONDS=120 RESTART_CHAOS_K15_FREEZE_NEW_SECONDS=150 node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts ... +# Loaded twin L1: 48 tasks, 20 runs plus 20 queued, 12-22 minute turns, 2 swaps 13 minutes apart, swap 1 graceful, swap 2 G3. About 78 minutes. +RESTART_CHAOS_DAYTONA_VCPU_BUDGET=48 RESTART_CHAOS_K15_KILL_OLD_SWAPS=2 RESTART_CHAOS_K15_KILL_OLD_AFTER_SECONDS=120 \ +RESTART_CHAOS_K15_FREEZE_SWAPS=2 RESTART_CHAOS_K15_FREEZE_NEW_SECONDS=150 RESTART_CHAOS_MIX_BANNER_REST_SECONDS=240 \ +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona --daytona-image $IMG --daytona-warm --daytona-cpu 1 --storage s3 \ + --workload staging-mix --mix-tasks 48 --mix-runs 20 --mix-queue 20 --mix-ratio opencode=34,codex=33,claude=33 --mix-turn-minutes 12-22 \ + --mix-hold-minutes 2 --mix-arrival-cutoff-minutes 0 --release-swap --release-swap-count 2 --release-swap-spacing-minutes 13 \ + --release-swap-overlap-seconds 120 --release-swap-settle-minutes 1 --seed 7301 +``` + +| Knob | Effect | +| --- | --- | +| `RESTART_CHAOS_K15_KILL_OLD_AFTER_SECONDS` | SIGKILL the outgoing server this many seconds into the overlap (at most the overlap) instead of a SIGTERM at its end | +| `RESTART_CHAOS_K15_KILL_OLD_SWAPS` | comma-separated swap numbers (1-based) the kill applies to; unset means every swap | +| `RESTART_CHAOS_K15_FREEZE_NEW_SECONDS` | SIGSTOP the incoming server for this long while the outgoing one stops, then SIGCONT | +| `RESTART_CHAOS_K15_FREEZE_SWAPS` | swap numbers the freeze applies to; unset means every swap | +| `RESTART_CHAOS_MIX_BANNER_REST_SECONDS` | a hands-off staging-mix task whose Retry banner has stood this long rests (it is still reported as a banner) instead of waiting out its budget | +| `RESTART_CHAOS_MIX_FIRST_SLOT_SECONDS` | place the first staging-mix fault slot this early (at most the slot spacing); the slot count stays the same, so a release swap never gets an extra swap | + +A run counts as a success only when `staging-mix-summary.md` shows every started task completed hands-off +(`completed hands-off / started` equal), `Retry banner shown: 0`, `Retry worked / Retry clicked` equal, `extra turns: 0` +and `duplicated side effects: 0`, the release-swap headline shows every task completed without human action, and the +Daytona cleanup left no sandbox (`left after deletion: none`). The `follow-up answered in a later run` line is not a +failure; read its waits. In L1 the queued tasks wait for a slot by design; U1 allows that wait (see "Universal checks"). + `--release-swap` runs the full default (6 swaps, 40 minutes apart). `--release-swap-short` runs 3 swaps 12 minutes apart with the full 510 s overlap, so two servers run for most of each cycle. Either takes the `--release-swap-*` options below, which override the preset. Workload sizing (turn length, concurrency) comes from the workload flags. @@ -1040,12 +1086,20 @@ disk), and `--concurrency × --daytona-cpu` must stay within 40 vCPU org limit with room for verification probes: a 10 vCPU org fails creates with HTTP 400 "Total CPU limit exceeded", so use 8 there and `--daytona-cpu 2`. -**Fleet safety.** The harness never stops, archives or deletes a Daytona -sandbox. Its Daytona client is a frozen wrapper that exposes only a -label-filtered `list`. `assertHarnessMayMutate` refuses `paperclip-create-*` -names (every plugin-created sandbox has one) and `fleet-sbx-*` snapshots, so a -future mutating path cannot act on fleet sandboxes either. Cleanup goes only -through Paperclip, by deleting the wave's own environment. +**Fleet safety.** The Daytona org is shared with the live Cloud fleet, and +every plugin-created sandbox, the fleet's included, is named +`paperclip-create-*`. The name therefore says nothing about who owns a +sandbox; the labels do. Each wave's isolated instance mints a fresh company +and environment UUID, and the Daytona plugin labels every sandbox it creates +with both. Cleanup first deletes the wave's own environment through Paperclip. +Then it deletes every sandbox that still carries both of this wave's labels +(archived setup sandboxes included, since Daytona does not delete archived +sandboxes on its own), and nothing else: the listing is filtered by both labels +on the server and checked again on every row, and `assertHarnessMayMutate` +refuses any sandbox without both labels and any `fleet-sbx-*` snapshot. The +only delete is `DELETE /sandbox/` for a row that passed those checks. A +wave sandbox still there after the settle window fails the wave +(`undeletedSandboxes`, and the warm-shutdown verdict). **Image matching.** The controller needs the image's own Linux amd64 `/usr/local/bin/paperclip-runnerd` (`PAPERCLIP_RUNNER_REMOTE_BINARY_PATH`) and @@ -1552,12 +1606,16 @@ click instead, and its issue reads also update the banner record. Both paths use So no task gets two clicks. `--mix-retry-during-wave` restores the old behaviour: Retries clicked during the wave, held until an overlap window as follow-ups are (`staging_mix_retry_in_overlap`). -**Results.** `staging-mix-results.json` and the first lines of `SUMMARY.md` give four lines, in this order: +**Results.** `staging-mix-results.json` and the first lines of `SUMMARY.md` give six lines, in this order: 1. `completed hands-off / started`. 2. `duplicated side effects: N tasks`, followed by `; evidence incomplete: M tasks` when a check could not read its evidence (such a check is never counted as a duplicate). 3. `Retry banner shown: N tasks`. 4. `Retry worked / Retry clicked`. +5. `extra turns`: runs that started after the task completed with no undelivered input. +6. `follow-up answered in a later run: n`, with the answering run's queued wait (p50 / max) and the time from the comment + to the answering run's start (p50 / max). These runs are required, not extra turns; the waits are shown because a long + one is a product concern (a person waits for an answer behind a full queue). A task counts as completed hands-off when all three hold, read from fresh data at the end of the wave: @@ -1575,6 +1633,18 @@ The summary also lists, by start time, every run that started after the task com the task completed" (`extraTurns`, with counts by wake reason in `extraTurnsByReason`). A run queued after completion that never started is listed on its own (`queuedAfterDone`). +The grader takes comment delivery from the server's own record and does not decide it again. A comment that the server +judged delivered to an earlier run never becomes a run: its wake is cancelled or coalesced. A run that the server +started from a wake that carries a person's comment posted after the task's first run started (`wakeCommentIds`, the +wake payload's `commentId`) is a required run. This includes the retry of a failed attempt for the same comment. OpenCode, +for example, takes no input mid-turn, so a comment posted during its turn is answered by the next run. Such runs are +listed under "Follow-ups the running turn did not receive, answered in a later run" (`followUpsAnsweredLater`, +`followUpWaits`). An extra turn is a run that started after completion from a wake that carries no comment. The +server's delivery rule (`task_history.read`, cited evidence, child-result reads) is covered by the server's own tests. + +`completion_once` stays the independent side-effect check. The first run that reported completion is always the +baseline. A later completion counts as a duplicate unless its run answered a comment as above. + The completion time comes from the product's own record, in this order: 1. the earliest accepted run result whose `reportedWorkDisposition` is `done`; @@ -1691,6 +1761,19 @@ its workload. They catch a stuck or misleading task that no scenario anticipated | U4 stale notice | a Retry or failed notice stays on a task after its latest run succeeded, longer than the bound or at rest | 60 s (`RESTART_CHAOS_U4_STALE_SECONDS`) | | U5 run budget | one task created more runs than the bound | 10 (`RESTART_CHAOS_U5_MAX_RUNS`) | +When the workload declares a queue (staging-mix `--mix-queue` above 0), a run that waits queued while its agent's +`maxConcurrentRuns` slots are all taken is a healthy queued run. U1 then allows it the declared queue bound, +ceil(queue / runs) × (longest turn + 5 min); for L1 (20 runs, 20 queued, 12–22 minute turns) that is 27 minutes. +Only queued time with a free slot of its agent counts toward the 5-minute queue bound, so a run left queued while +its agent has a free slot still fails. Slot use comes from `wake-evidence/agent-slots.json` (every run's start and +finish and each agent's effective cap as the server computes it, `maxConcurrentRunsSql`, so a configured 100 is saved +as the server's limit of 50; captured at wave end). Without that file (older waves) only the declared bound applies. + +A parent whose turn ended `yielded` or `blocked` and that waits, with no run, until its subtasks complete is in a +named wait (`waitingOnChildren`): R5 and U1 do not count it as a stall. The observer keeps watching such a parent +while any subtask is open. When an older record rested during that wait and the parent then completed through its +`issue_children_completed` wake, grading takes the completion from the wake evidence. + U1, U2, U4 and U5 fail the task (signature `U:...`). U3 is a warning: it is reported in each grade (`grade.universal`), the ledger (`universal`, `universalWarnings`) and the wave's `SUMMARY.md`, and never fails a task. U3 reads run error text and recovery-action causes from the wave database at diff --git a/tests/restart-chaos/agent-slots.ts b/tests/restart-chaos/agent-slots.ts new file mode 100644 index 00000000000..a13ed7a9c9e --- /dev/null +++ b/tests/restart-chaos/agent-slots.ts @@ -0,0 +1,35 @@ +import type { SlotFree } from "./universal.js"; + +export const AGENT_SLOTS_FILE = "agent-slots.json"; + +export interface AgentSlots { + agents: Array<{ agentId: string; maxConcurrentRuns: number | null }>; + runs: Array<{ runId: string; agentId: string; startedAt: number | null; finishedAt: number | null }>; +} + +export function slotFreeFrom(slots: AgentSlots | null): SlotFree { + if (!slots) return () => null; + const caps = new Map(slots.agents.map(agent => [agent.agentId, agent.maxConcurrentRuns])); + const agentOf = new Map(slots.runs.map(run => [run.runId, run.agentId])); + const byAgent = new Map(); + for (const run of slots.runs) byAgent.set(run.agentId, [...byAgent.get(run.agentId) ?? [], run]); + return (runId, at) => { + const agentId = agentOf.get(runId); + const cap = agentId ? caps.get(agentId) : undefined; + if (!agentId || typeof cap !== "number" || cap < 1) return null; + const running = (byAgent.get(agentId) ?? []).filter(run => run.runId !== runId && run.startedAt !== null && run.startedAt <= at + && (run.finishedAt === null || run.finishedAt > at)).length; + return running < cap; + }; +} + +export function agentSlotsFromRows(agents: ReadonlyArray>, runs: ReadonlyArray>): AgentSlots { + const ms = (value: unknown) => value === null || value === undefined ? null : Number(value); + return { + agents: agents.map(agent => { + const cap = Number(agent.maxConcurrentRuns); + return { agentId: String(agent.id), maxConcurrentRuns: Number.isInteger(cap) && cap > 0 ? cap : null }; + }), + runs: runs.filter(run => run.agentId).map(run => ({ runId: String(run.id), agentId: String(run.agentId), startedAt: ms(run.startedAt), finishedAt: ms(run.finishedAt) })), + }; +} diff --git a/tests/restart-chaos/child-wait.test.ts b/tests/restart-chaos/child-wait.test.ts new file mode 100644 index 00000000000..a81b1db9281 --- /dev/null +++ b/tests/restart-chaos/child-wait.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; +import { withChildWaits } from "./child-wait.js"; +import { gradeTask } from "./grader.js"; +import type { WakeEvidenceFile } from "./staging-mix-retry.js"; +import type { RunSample, TaskRecord, TaskSample } from "./types.js"; + +const T0 = Date.parse("2026-11-13T05:37:21Z"); +const SEC = 1_000; +const iso = (offset: number) => new Date(T0 + offset).toISOString(); + +const run = (id: string, status: string, started: number, finished: number | null): RunSample => ({ id, status, createdAt: iso(started), startedAt: iso(started), + finishedAt: finished === null ? null : iso(finished), errorCode: null, retryOfRunId: null, wakeCommentIds: [], livenessState: null, + lastEvidenceAt: status === "running" ? iso(started) : null }); +const sample = (at: number, runs: RunSample[], issueStatus = "in_progress"): TaskSample => ({ at: T0 + at, reachable: true, serverReady: true, issueStatus, + interruption: null, activeRecoveryAction: false, executionBlocker: false, scheduledRetry: false, runs }); + +function parentRestedWaitingOnSubtasks(): TaskRecord { + const samples: TaskSample[] = []; + for (let at = 0; at < 2_000 * SEC; at += 10 * SEC) samples.push(sample(at, [{ ...run("r1", "running", 0, null), lastEvidenceAt: iso(at) }])); + for (let at = 2_010 * SEC; at <= 2_210 * SEC; at += 10 * SEC) samples.push(sample(at, [run("r1", "succeeded", 0, 2_000 * SEC)])); + return { plan: { taskId: "t001", index: 0, workload: "SM", adapter: "opencode", fault: "K13", fireAfterMs: 0, holdMs: 0, followUpAfterMs: null, nonce: "90a3899c" }, + issueId: "issue-1", identifier: "CHA-1", createdAt: T0, launchError: null, samples, retry: null, restAt: T0 + 2_010 * SEC, finalStatus: "in_progress", + deadlineAt: T0 + 7_200 * SEC, timedOut: false, followUp: null, faultIds: [], sweep: { activeRuns: [], runningWorkspaceOperations: [], activeRecoveryAction: false, error: null }, + preFaultFiles: null, lostFiles: null }; +} + +const evidence = (withChildCompletion: boolean): WakeEvidenceFile => ({ + completedAt: T0 + 2_871 * SEC, + runs: [{ runId: "r1", createdAt: T0, startedAt: T0, finishedAt: T0 + 2_000 * SEC, invocationSource: "assignment", triggerDetail: "system", wakeupRequestId: "w1", + acceptedDisposition: "yielded", acceptedAt: T0 + 1_999 * SEC }, + ...(withChildCompletion ? [{ runId: "r3", createdAt: T0 + 2_667 * SEC, startedAt: T0 + 2_820 * SEC, finishedAt: T0 + 2_871 * SEC, invocationSource: "automation", + triggerDetail: "system", wakeupRequestId: "w3", acceptedDisposition: "done", acceptedAt: T0 + 2_870 * SEC }] : [])], + wakes: [{ id: "w1", reason: "issue_assigned", source: "assignment", triggerDetail: "system", requestedByActorType: "user", requestedByActorId: "local-board", createdAt: T0 }, + ...(withChildCompletion ? [{ id: "w3", reason: "issue_children_completed", source: "automation", triggerDetail: "system", requestedByActorType: "agent", + requestedByActorId: "a1", createdAt: T0 + 2_648 * SEC }] : [])], +}); + +const pass = { verdict: "pass" as const, checks: [] }; + +describe("a parent waiting on its subtasks (w-20261113T052703-s7301 t001)", () => { + it("is a named wait, and the parent completed by its children-completed wake rests done", () => { + const before = gradeTask(parentRestedWaitingOnSubtasks(), [], 4_345 * SEC, pass); + expect([before.rules.R1.detail, before.rules.R3.verdict, before.rules.R5.verdict, before.universal!.U1.verdict]) + .toEqual(["rested_not_done (status in_progress, no Retry offered)", "fail", "fail", "fail"]); + const record = withChildWaits(parentRestedWaitingOnSubtasks(), evidence(true)); + expect(record).toMatchObject({ finalStatus: "done", restAt: T0 + 2_871 * SEC }); + expect(record.samples.filter(entry => entry.waitingOnChildren).length).toBe(21); + const after = gradeTask(record, [], 4_345 * SEC, pass); + expect([after.rules.R1.verdict, after.rules.R3.verdict, after.rules.R5.verdict, after.universal!.U1.verdict, after.pass]).toEqual(["pass", "n/a", "pass", "pass", true]); + }); + + it("changes nothing when no children-completed wake followed the parent's yielded turn", () => { + const record = parentRestedWaitingOnSubtasks(); + expect(withChildWaits(record, evidence(false))).toBe(record); + expect(withChildWaits(record, null)).toBe(record); + expect(gradeTask(withChildWaits(record, evidence(false)), [], 4_345 * SEC, pass).pass).toBe(false); + }); +}); diff --git a/tests/restart-chaos/child-wait.ts b/tests/restart-chaos/child-wait.ts new file mode 100644 index 00000000000..1b3383fdebd --- /dev/null +++ b/tests/restart-chaos/child-wait.ts @@ -0,0 +1,38 @@ +import type { TaskRecord, TaskSample } from "./types.js"; +import type { WakeEvidenceFile } from "./staging-mix-retry.js"; + +const ACTIVE = new Set(["queued", "running", "scheduled_retry"]); +const WAITING_DISPOSITIONS = new Set(["yielded", "blocked"]); +const CHILDREN_COMPLETED = "issue_children_completed"; + +function childWaitWindows(evidence: WakeEvidenceFile): Array<{ from: number; until: number }> { + const completions = (evidence.wakes ?? []).filter(wake => wake.reason === CHILDREN_COMPLETED && typeof wake.createdAt === "number") + .map(wake => wake.createdAt!).sort((a, b) => a - b); + return (evidence.runs ?? []).flatMap(run => { + if (!WAITING_DISPOSITIONS.has(run.acceptedDisposition ?? "") || typeof run.finishedAt !== "number") return []; + const until = completions.find(at => at >= run.finishedAt!); + return until === undefined ? [] : [{ from: run.finishedAt, until }]; + }); +} + +function idle(sample: TaskSample): boolean { + return sample.reachable && !["done", "cancelled"].includes(sample.issueStatus ?? "") && !(sample.runs ?? []).some(run => ACTIVE.has(run.status)); +} + +export function withChildWaits(record: TaskRecord, evidence: WakeEvidenceFile | null): TaskRecord { + if (!evidence) return record; + const windows = childWaitWindows(evidence); + if (!windows.length) return record; + const waiting = (sample: TaskSample) => idle(sample) && windows.some(window => sample.at >= window.from && sample.at <= window.until); + const samples = record.samples.map(sample => waiting(sample) ? { ...sample, waitingOnChildren: true } : sample); + const last = [...samples].reverse().find(sample => sample.reachable); + const wakes = new Map((evidence.wakes ?? []).map(wake => [wake.id, wake])); + const finisher = (evidence.runs ?? []).find(run => run.acceptedDisposition === "done" && run.wakeupRequestId + && wakes.get(run.wakeupRequestId)?.reason === CHILDREN_COMPLETED); + const completedAt = evidence.completedAt ?? null; + const restedWaiting = record.finalStatus !== "done" && record.restAt !== null && last?.waitingOnChildren === true; + if (restedWaiting && finisher && completedAt !== null && completedAt > record.restAt!) { + return { ...record, samples, finalStatus: "done", restAt: completedAt }; + } + return { ...record, samples }; +} diff --git a/tests/restart-chaos/daytona-cleanup.test.ts b/tests/restart-chaos/daytona-cleanup.test.ts new file mode 100644 index 00000000000..773f7e96cab --- /dev/null +++ b/tests/restart-chaos/daytona-cleanup.test.ts @@ -0,0 +1,48 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { finishDaytonaCleanup } from "./daytona-cleanup.js"; +import type { DaytonaSweepResult, WaveCleanup } from "./daytona-sweep.js"; + +const sweep: DaytonaSweepResult = { companyId: "c", environmentId: "e", checkedAt: "now", liveSandboxes: ["s1"], orphanSandboxes: [], heldLeases: [], + retainedLeases: [], archivedSandboxes: ["s1"], error: null }; +const clean: WaveCleanup = { environmentDeleted: true, environmentDeleteBlocked: null, remainingSandboxes: [], archivedSandboxes: ["s1:archived"], + deletedSandboxes: ["s1:archived"], undeletedSandboxes: [], errors: [] }; + +async function run(cleanup: WaveCleanup, saved: DaytonaSweepResult | null) { + const waveDir = await mkdtemp(path.join(os.tmpdir(), "daytona-cleanup-")); + const kinds: string[] = [], lines: string[] = []; + let swept = 0; + try { + const outcome = await finishDaytonaCleanup({ sweeper: { sweep: async () => { swept += 1; return sweep; }, cleanUp: async () => cleanup }, sweep: saved, waveDir, + timeline: { add: kind => { kinds.push(kind); } }, log: line => { lines.push(line); } }); + const file = JSON.parse(await readFile(path.join(waveDir, "sandbox-sweep.json"), "utf8")); + return { outcome, kinds, lines, swept, file }; + } finally { + await rm(waveDir, { recursive: true, force: true }); + } +} + +describe("wave-end Daytona cleanup", () => { + it("passes a wave whose sandboxes were all deleted, and sweeps first when the wave was interrupted", async () => { + const { outcome, kinds, lines, swept, file } = await run(clean, null); + expect(outcome).toMatchObject({ failed: false, remotePending: false, error: null, sweep }); + expect(swept).toBe(1); + expect(kinds).toEqual(["daytona_cleanup"]); + expect(lines[0]).toContain("deleted by the harness (this wave's labels only): s1:archived; left after deletion: none"); + expect(file).toEqual({ sweep, cleanup: clean }); + }); + + it("fails the wave for a sandbox left after deletion, a blocked environment delete or a still-running sandbox, and reports errors", async () => { + expect((await run({ ...clean, undeletedSandboxes: ["s1:archived"] }, sweep)).outcome).toMatchObject({ failed: true, remotePending: true }); + const blocked = await run({ ...clean, environmentDeleted: false, environmentDeleteBlocked: "409" }, sweep); + expect(blocked.outcome).toMatchObject({ failed: true, remotePending: true }); + expect(blocked.kinds).toEqual(["daytona_cleanup", "daytona_environment_delete_blocked"]); + expect(blocked.swept).toBe(0); + expect((await run({ ...clean, remainingSandboxes: ["s2:started"] }, sweep)).outcome.failed).toBe(true); + const errored = await run({ ...clean, errors: ["sandbox delete s1: boom"] }, sweep); + expect(errored.outcome).toMatchObject({ failed: false, remotePending: true }); + expect(errored.outcome.error?.message).toBe("Daytona cleanup incomplete: sandbox delete s1: boom"); + }); +}); diff --git a/tests/restart-chaos/daytona-cleanup.ts b/tests/restart-chaos/daytona-cleanup.ts new file mode 100644 index 00000000000..8dfb0ee2b20 --- /dev/null +++ b/tests/restart-chaos/daytona-cleanup.ts @@ -0,0 +1,35 @@ +import { writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { DaytonaSweepResult, DaytonaWaveSweeper, WaveCleanup } from "./daytona-sweep.js"; + +export interface DaytonaCleanupOutcome { + sweep: DaytonaSweepResult; + cleanup: WaveCleanup; + failed: boolean; + remotePending: boolean; + error: Error | null; +} + +const list = (values: readonly string[]) => values.join(", ") || "none"; + +export function daytonaCleanupLine(sweep: DaytonaSweepResult, cleanup: WaveCleanup): string { + return `Daytona R6 sweep: ${sweep.liveSandboxes.length} live, ${sweep.archivedSandboxes.length} archived after release, ` + + `${sweep.orphanSandboxes.length} orphaned, ${sweep.heldLeases.length} held leases${sweep.error ? `, error ${sweep.error}` : ""}; ` + + `environment deleted: ${cleanup.environmentDeleted}${cleanup.environmentDeleteBlocked ? " (still blocked by a pending sandbox cleanup)" : ""}; ` + + `still running after cleanup: ${list(cleanup.remainingSandboxes)}; archived: ${list(cleanup.archivedSandboxes)}; ` + + `deleted by the harness (this wave's labels only): ${list(cleanup.deletedSandboxes)}; left after deletion: ${list(cleanup.undeletedSandboxes)}`; +} + +export async function finishDaytonaCleanup(input: { sweeper: Pick; sweep: DaytonaSweepResult | null; waveDir: string; + timeline: { add(kind: string, data: Record): void }; log?: (line: string) => void }): Promise { + const sweep = input.sweep ?? await input.sweeper.sweep(0); + const cleanup = await input.sweeper.cleanUp(); + input.timeline.add("daytona_cleanup", cleanup); + (input.log ?? console.info)(daytonaCleanupLine(sweep, cleanup)); + if (cleanup.environmentDeleteBlocked) input.timeline.add("daytona_environment_delete_blocked", { detail: cleanup.environmentDeleteBlocked }); + const failed = Boolean(cleanup.remainingSandboxes.length || cleanup.undeletedSandboxes.length || cleanup.environmentDeleteBlocked); + await writeFile(path.join(input.waveDir, "sandbox-sweep.json"), `${JSON.stringify({ sweep, cleanup }, null, 2)}\n`, { mode: 0o600 }) + .catch(error => input.timeline.add("sandbox_sweep_write_failed", { error: String(error) })); + return { sweep, cleanup, failed, remotePending: failed || cleanup.errors.length > 0, + error: cleanup.errors.length ? new Error(`Daytona cleanup incomplete: ${cleanup.errors.join("; ")}`) : null }; +} diff --git a/tests/restart-chaos/daytona-sweep.test.ts b/tests/restart-chaos/daytona-sweep.test.ts index b2e6f91a9cc..e28282f7103 100644 --- a/tests/restart-chaos/daytona-sweep.test.ts +++ b/tests/restart-chaos/daytona-sweep.test.ts @@ -1,9 +1,10 @@ import { describe, expect, it } from "vitest"; import { - assertHarnessMayMutate, DaytonaWaveSweeper, isProtectedSandbox, leaksForTask, readOnlyDaytona, reconcileSandboxes, selectWaveSandboxes, - PENDING_CLEANUP_RETRY_MS, R6_SETTLE_MS, sweepHasLeaks, unattributedLeaks, type DaytonaSandboxRow, type DaytonaSweepResult, + assertHarnessMayMutate, daytonaRestDelete, DaytonaWaveSweeper, isProtectedSandbox, leaksForTask, readOnlyDaytona, reconcileSandboxes, selectWaveSandboxes, + PENDING_CLEANUP_RETRY_MS, R6_SETTLE_MS, SANDBOX_DELETE_MIN_TIMEOUT_MS, sweepHasLeaks, unattributedLeaks, type DaytonaSandboxRow, type DaytonaSweepResult, } from "./daytona-sweep.js"; import { ENVIRONMENT_CLEANUP_RETRY_MS } from "../../server/src/services/environment-cleanup-state.js"; +import { gradeWarmShutdown } from "./warm-shutdown.js"; describe("R6 settle window", () => { it("covers the product's pending-cleanup retry backoff plus a sweep, capped at 8 minutes", () => { @@ -16,6 +17,7 @@ describe("R6 settle window", () => { describe("cleanup through Paperclip", () => { const COMPANY = "11111111-1111-4111-8111-111111111111"; const ENVIRONMENT = "22222222-2222-4222-8222-222222222222"; + const OTHER_COMPANY = "33333333-3333-4333-8333-333333333333"; const listing = { list: () => (async function* () {})() }; it("treats a 409 pending-cleanup environment delete as blocked, not a harness error, and retries it", async () => { @@ -29,6 +31,76 @@ describe("cleanup through Paperclip", () => { expect(calls).toBeGreaterThan(1); }); + const waveRows = (states: string[], extra: Partial = {}): DaytonaSandboxRow[] => states.map(state => + ({ id: state, name: `paperclip-create-${state}`, state, labels: { "paperclip-company-id": COMPANY, "paperclip-environment-id": ENVIRONMENT }, ...extra })); + const deletingDaytona = (rows: DaytonaSandboxRow[]) => { + const removed: string[] = []; + const client = readOnlyDaytona({ list: () => (async function* () { yield* rows.map(row => ({ ...row })); })() }); + const removeSandbox = async (id: string) => { + removed.push(id); + const row = rows.find(candidate => candidate.id === id); + if (row) row.state = "destroyed"; + }; + return { client, removeSandbox, removed }; + }; + + it("deletes the wave's archived setup sandboxes and every other sandbox carrying both wave labels, and leaves nothing behind", async () => { + const rows = [...waveRows(["started", "archiving", "archived", "destroyed"]), + { id: "fleet", name: "paperclip-create-fleet", state: "archived", labels: { "paperclip-company-id": OTHER_COMPANY, "paperclip-environment-id": ENVIRONMENT } }, + { id: "other-env", name: "paperclip-create-x", state: "archived", labels: { "paperclip-company-id": COMPANY, "paperclip-environment-id": OTHER_COMPANY } }]; + const { client, removeSandbox, removed } = deletingDaytona(rows); + const sweeper = new DaytonaWaveSweeper({ apiKey: "unused", companyId: COMPANY, environmentId: ENVIRONMENT, api: { delete: async () => undefined } as never, removeSandbox }); + (sweeper as unknown as { client: unknown }).client = client; + const result = await sweeper.cleanUp(0, 1_000); + expect(result.remainingSandboxes).toEqual(["started:started"]); + expect(result.archivedSandboxes).toEqual(["archiving:archiving", "archived:archived"]); + expect(removed.sort()).toEqual(["archived", "archiving", "started"]); + expect(result.deletedSandboxes.sort()).toEqual(["archived:archived", "archiving:archiving", "started:started"]); + expect(result.undeletedSandboxes).toEqual([]); + expect(rows.find(row => row.id === "fleet")!.state).toBe("archived"); + expect(rows.find(row => row.id === "other-env")!.state).toBe("archived"); + }); + + it("fails cleanup when a wave sandbox survives deletion, and never deletes a fleet snapshot even with the wave's labels", async () => { + const rows = [...waveRows(["archived"]), ...waveRows(["fleet"], { snapshot: "fleet-sbx-9", state: "archived" })]; + const { client, removeSandbox, removed } = deletingDaytona(rows); + const sweeper = new DaytonaWaveSweeper({ apiKey: "unused", companyId: COMPANY, environmentId: ENVIRONMENT, api: { delete: async () => undefined } as never, removeSandbox }); + (sweeper as unknown as { client: unknown }).client = client; + const result = await sweeper.cleanUp(0, 1_000); + expect(removed).toEqual(["archived"]); + expect(result.undeletedSandboxes).toEqual(["fleet:archived"]); + expect(result.errors.join(" ")).toMatch(/sandbox delete fleet: Refusing to change protected/); + expect(gradeWarmShutdown({ cleanup: result, serverLog: "" })).toMatchObject({ verdict: "fail", leakedSandboxes: ["fleet:archived"] }); + }); + + it("sends the Daytona delete for one sandbox id with the wave's key", async () => { + const calls: Array<{ url: string; init: RequestInit | undefined }> = []; + const remove = daytonaRestDelete("key-1", "https://daytona.test/api", (async (url: string, init?: RequestInit) => { + calls.push({ url, init }); + return new Response(null, { status: url.endsWith("gone") ? 404 : url.endsWith("bad") ? 500 : 200 }); + }) as typeof fetch); + const signal = new AbortController().signal; + await remove("abc", signal); + await remove("gone", signal); + await expect(remove("bad", signal)).rejects.toThrow(/returned 500/); + expect(calls[0]).toEqual({ url: "https://daytona.test/api/sandbox/abc", init: { method: "DELETE", headers: { Authorization: "Bearer key-1" }, signal } }); + }); + + it("aborts a hung delete at the cleanup deadline and reports the sandbox as left", async () => { + const rows = waveRows(["archived"]); + const client = readOnlyDaytona({ list: () => (async function* () { yield* rows.map(row => ({ ...row })); })() }); + const removeSandbox = (_id: string, signal: AbortSignal) => new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => reject(new Error("delete aborted")), { once: true }); + }); + const sweeper = new DaytonaWaveSweeper({ apiKey: "unused", companyId: COMPANY, environmentId: ENVIRONMENT, api: { delete: async () => undefined } as never, removeSandbox }); + (sweeper as unknown as { client: unknown }).client = client; + const started = Date.now(); + const result = await sweeper.cleanUp(0, 1_000); + expect(Date.now() - started).toBeLessThan(SANDBOX_DELETE_MIN_TIMEOUT_MS + 3_000); + expect(result.undeletedSandboxes).toEqual(["archived:archived"]); + expect(result.errors.join(" ")).toMatch(/sandbox delete archived: delete aborted/); + }); + it("reports other environment delete failures as errors", async () => { const api = { delete: async () => { throw new Error("DELETE returned 500: boom"); } }; const sweeper = new DaytonaWaveSweeper({ apiKey: "unused", companyId: COMPANY, environmentId: ENVIRONMENT, api: api as never }); @@ -79,13 +151,13 @@ describe("wave sandbox selection is read-only and exact", () => { expect(() => readOnlyDaytona({} as never)).toThrow(/no list\(\)/); }); - it("hard-refuses fleet sandboxes and anything not labelled for the wave, even with matching labels", () => { + it("hard-refuses fleet snapshots and anything not labelled for the wave; a plugin-created name with the wave's labels is the wave's own", () => { const waveLabels = { "paperclip-company-id": COMPANY, "paperclip-environment-id": ENVIRONMENT }; - expect(isProtectedSandbox({ name: "paperclip-create-1" })).toBe(true); expect(isProtectedSandbox({ name: "x", snapshot: "fleet-sbx-2" })).toBe(true); - expect(() => assertHarnessMayMutate(labelled("a", waveLabels, { name: "paperclip-create-1" }), wave)).toThrow(/protected/); expect(() => assertHarnessMayMutate(labelled("b", waveLabels, { name: "probe", snapshot: "fleet-sbx-2" }), wave)).toThrow(/protected/); expect(() => assertHarnessMayMutate(labelled("c", {}, { name: "probe" }), wave)).toThrow(/not labelled for this wave/); + expect(() => assertHarnessMayMutate(labelled("e", { "paperclip-company-id": OTHER }, { name: "paperclip-create-1" }), wave)).toThrow(/not labelled for this wave/); + expect(() => assertHarnessMayMutate(labelled("a", waveLabels, { name: "paperclip-create-1" }), wave)).not.toThrow(); expect(() => assertHarnessMayMutate(labelled("d", waveLabels, { name: "probe" }), wave)).not.toThrow(); }); }); diff --git a/tests/restart-chaos/daytona-sweep.ts b/tests/restart-chaos/daytona-sweep.ts index d1d434ff51e..2e31241faee 100644 --- a/tests/restart-chaos/daytona-sweep.ts +++ b/tests/restart-chaos/daytona-sweep.ts @@ -2,12 +2,6 @@ * R6 for `--env daytona`: reconcile the Daytona sandboxes this wave created * against the instance's environment leases and report leaks. * - * The Daytona org is shared with the live Cloud fleet, so this module NEVER - * changes a sandbox. Its client exposes only `list`; stop, archive and delete - * do not exist here. Cleanup goes only through Paperclip: deleting the wave's - * own environment, which acts on the leases in the wave's isolated database. - * Leaks are reported for a person, and expire through the environment's ttl. - * * Wave ownership comes from the labels the Daytona plugin puts on every * sandbox it creates. Each wave's isolated instance mints a fresh company and * environment UUID, and a row is selected only when both labels equal them. @@ -129,7 +123,6 @@ const CLEANUP_SWEEP_SLACK_MS = 90_000; /** R6 waits this long for leases and sandboxes to settle before calling them leaks. */ export const R6_SETTLE_MS = Math.min(8 * 60_000, PENDING_CLEANUP_RETRY_MS + CLEANUP_SWEEP_SLACK_MS); -/** The only Daytona operation the harness may perform. */ export interface ReadOnlyDaytona { list(query: { labels: Record }): AsyncIterable } /** @@ -142,22 +135,34 @@ export function readOnlyDaytona(client: { list(query?: { labels?: Record }) => client.list(query) }); } -/** Fleet sandboxes the harness must never act on, even if their labels matched. */ export function isProtectedSandbox(sandbox: Pick): boolean { - return (sandbox.name ?? "").startsWith("paperclip-create-") || (sandbox.snapshot ?? "").startsWith("fleet-sbx-"); + return (sandbox.snapshot ?? "").startsWith("fleet-sbx-"); } -/** - * Hard guard for any future mutating path. It refuses `paperclip-create-*` - * names and `fleet-sbx-*` snapshots outright, and anything not carrying both - * of the wave's UUID labels. Every plugin-created sandbox is - * `paperclip-create-*`, so the harness can never mutate one directly. - */ export function assertHarnessMayMutate(sandbox: DaytonaSandboxRow, wave: { companyId: string; environmentId: string }): void { if (isProtectedSandbox(sandbox)) throw new Error(`Refusing to change protected Daytona sandbox ${sandbox.id} (${sandbox.name ?? sandbox.snapshot})`); if (selectWaveSandboxes([sandbox], wave).length !== 1) throw new Error(`Refusing to change Daytona sandbox ${sandbox.id}: it is not labelled for this wave`); } +export type SandboxRemove = (sandboxId: string, signal: AbortSignal) => Promise; +export type WaveSandboxDelete = (sandbox: DaytonaSandboxRow, signal: AbortSignal) => Promise; + +export const SANDBOX_DELETE_MIN_TIMEOUT_MS = 5_000; + +export function waveSandboxDeleter(wave: { companyId: string; environmentId: string }, remove: SandboxRemove): WaveSandboxDelete { + return Object.freeze(async (sandbox: DaytonaSandboxRow, signal: AbortSignal) => { + assertHarnessMayMutate(sandbox, wave); + await remove(sandbox.id, signal); + }); +} + +export function daytonaRestDelete(apiKey: string, apiUrl = "https://app.daytona.io/api", request: typeof fetch = fetch): SandboxRemove { + return async (sandboxId, signal) => { + const response = await request(`${apiUrl}/sandbox/${encodeURIComponent(sandboxId)}`, { method: "DELETE", headers: { Authorization: `Bearer ${apiKey}` }, signal }); + if (!response.ok && response.status !== 404) throw new Error(`DELETE sandbox ${sandboxId} returned ${response.status}`); + }; +} + const UUID = /^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/i; /** Keeps only rows whose company AND environment labels equal this wave's UUIDs. */ @@ -169,7 +174,12 @@ export function selectWaveSandboxes(rows: readonly DaytonaSandboxRow[], wave: { export class DaytonaWaveSweeper { private client: ReadOnlyDaytona | null = null; - constructor(private readonly input: { apiKey: string; companyId: string; environmentId: string; api: ChaosApi }) {} + private readonly deleteSandbox: WaveSandboxDelete; + constructor(private readonly input: { apiKey: string; companyId: string; environmentId: string; api: ChaosApi; + removeSandbox?: SandboxRemove }) { + this.deleteSandbox = waveSandboxDeleter({ companyId: input.companyId, environmentId: input.environmentId }, + input.removeSandbox ?? daytonaRestDelete(input.apiKey)); + } private async daytona(): Promise { this.client ??= readOnlyDaytona(await createRemoteFixtureClient(this.input.apiKey) as unknown as Parameters[0]); @@ -205,13 +215,7 @@ export class DaytonaWaveSweeper { } } - /** - * Cleanup through Paperclip only: delete the wave's own environment (with its - * reusable leases). Then list, read-only, what is still not destroyed or - * archived, for the report. No sandbox is changed by the harness itself. - */ - async cleanUp(settleMs = 90_000, deleteWindowMs = R6_SETTLE_MS): Promise<{ environmentDeleted: boolean; environmentDeleteBlocked: string | null; - remainingSandboxes: string[]; errors: string[] }> { + async cleanUp(settleMs = 90_000, deleteWindowMs = R6_SETTLE_MS): Promise { const errors: string[] = []; let environmentDeleted = false; let environmentDeleteBlocked: string | null = null; @@ -226,14 +230,16 @@ export class DaytonaWaveSweeper { else errors.push(`environment delete: ${message}`); } let remainingSandboxes: string[] = []; + let archivedSandboxes: string[] = []; // Provider teardown is asynchronous; give it a bounded settle before reporting. const deadline = Date.now() + settleMs; for (;;) { try { - remainingSandboxes = (await this.listWaveSandboxes()) - .filter(sandbox => !GONE_SANDBOX_STATES.has(String(sandbox.state ?? "").toLowerCase()) - && !ARCHIVED_SANDBOX_STATES.has(String(sandbox.state ?? "").toLowerCase())) + const listed = await this.listWaveSandboxes(); + const state = (sandbox: DaytonaSandboxRow) => String(sandbox.state ?? "").toLowerCase(); + remainingSandboxes = listed.filter(sandbox => !GONE_SANDBOX_STATES.has(state(sandbox)) && !ARCHIVED_SANDBOX_STATES.has(state(sandbox))) .map(sandbox => `${sandbox.id}:${sandbox.state}`); + archivedSandboxes = listed.filter(sandbox => ARCHIVED_SANDBOX_STATES.has(state(sandbox))).map(sandbox => `${sandbox.id}:${sandbox.state}`); } catch (error) { errors.push(`sandbox listing: ${error instanceof Error ? error.message : String(error)}`); break; @@ -241,6 +247,46 @@ export class DaytonaWaveSweeper { if (!remainingSandboxes.length || Date.now() >= deadline) break; await delay(10_000); } - return { environmentDeleted, environmentDeleteBlocked, remainingSandboxes, errors }; + const removal = await this.deleteWaveSandboxes(settleMs, errors); + return { environmentDeleted, environmentDeleteBlocked, remainingSandboxes, archivedSandboxes, ...removal, errors }; + } + + private async deleteWaveSandboxes(settleMs: number, errors: string[]): Promise> { + const deleted = new Set(); + const failures = new Map(); + let undeleted: DaytonaSandboxRow[] = []; + const deadline = Date.now() + settleMs; + for (let attempt = 0; ; attempt += 1) { + try { + undeleted = (await this.listWaveSandboxes()).filter(sandbox => !GONE_SANDBOX_STATES.has(String(sandbox.state ?? "").toLowerCase())); + } catch (error) { + errors.push(`sandbox listing for deletion: ${error instanceof Error ? error.message : String(error)}`); + return { deletedSandboxes: [...deleted], undeletedSandboxes: ["unlisted"] }; + } + if (!undeleted.length || (attempt > 0 && Date.now() >= deadline)) break; + const pass = AbortSignal.timeout(Math.max(SANDBOX_DELETE_MIN_TIMEOUT_MS, deadline - Date.now())); + for (const sandbox of undeleted) { + try { + await this.deleteSandbox(sandbox, pass); + deleted.add(`${sandbox.id}:${sandbox.state}`); + failures.delete(sandbox.id); + } catch (error) { + failures.set(sandbox.id, error instanceof Error ? error.message : String(error)); + } + } + await delay(Math.min(10_000, Math.max(0, deadline - Date.now()))); + } + for (const [id, message] of failures) if (undeleted.some(sandbox => sandbox.id === id)) errors.push(`sandbox delete ${id}: ${message}`); + return { deletedSandboxes: [...deleted], undeletedSandboxes: undeleted.map(sandbox => `${sandbox.id}:${sandbox.state}`) }; } } + +export type WaveCleanup = { + environmentDeleted: boolean; + environmentDeleteBlocked: string | null; + remainingSandboxes: string[]; + archivedSandboxes: string[]; + deletedSandboxes: string[]; + undeletedSandboxes: string[]; + errors: string[]; +}; diff --git a/tests/restart-chaos/follow-up-delivery.ts b/tests/restart-chaos/follow-up-delivery.ts new file mode 100644 index 00000000000..0e907b116ec --- /dev/null +++ b/tests/restart-chaos/follow-up-delivery.ts @@ -0,0 +1,64 @@ +type Json = Record; + +export interface DeliveryRun { + runId: string | null; + createdAt: number | null; + startedAt?: number | null; + wakeupRequestId: string | null; + contextSnapshot?: unknown; +} + +export interface DeliveryWake { id: string | null; createdAt?: number | null; payload?: unknown } + +export interface LaterAnswer { + runId: string; + commentIds: string[]; + postedAt: number | null; + queuedAt: number | null; + startedAt: number; +} + +const strings = (value: unknown): string[] => Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") + : typeof value === "string" ? [value] : []; +const object = (value: unknown): Json => value && typeof value === "object" && !Array.isArray(value) ? value as Json : {}; + +export function wakeInputCommentIds(run: Pick, wake: DeliveryWake | undefined): string[] { + const context = object(run.contextSnapshot); + const payload = object(wake?.payload); + const wakeContext = object(payload._paperclipWakeContext); + return [...new Set([...strings(context.wakeCommentIds), ...strings(context.wakeCommentId), ...strings(object(context.paperclipWake).commentIds), + ...strings(payload.commentId), ...strings(wakeContext.wakeCommentIds), ...strings(wakeContext.wakeCommentId)])]; +} + +export function laterAnswers(evidence: { runs?: readonly DeliveryRun[]; wakes?: readonly DeliveryWake[] } | null): Map { + const answers = new Map(); + if (!evidence) return answers; + const wakes = new Map((evidence.wakes ?? []).map(wake => [wake.id, wake])); + const started = (evidence.runs ?? []).filter((run): run is DeliveryRun & { runId: string; startedAt: number } => Boolean(run.runId) && typeof run.startedAt === "number") + .sort((a, b) => a.startedAt - b.startedAt); + const firstStart = started[0]?.startedAt ?? null; + started.forEach((run, index) => { + const wake = run.wakeupRequestId ? wakes.get(run.wakeupRequestId) : undefined; + const owed = wakeInputCommentIds(run, wake); + const postedAt = typeof wake?.createdAt === "number" ? wake.createdAt : null; + const postedDuringWork = postedAt !== null && firstStart !== null ? postedAt >= firstStart : index > 0; + if (index > 0 && postedDuringWork && owed.length) { + answers.set(run.runId, { runId: run.runId, commentIds: owed, postedAt, queuedAt: run.createdAt, startedAt: run.startedAt }); + } + }); + return answers; +} + +function percentile(values: readonly number[], fraction: number): number | null { + if (!values.length) return null; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(fraction * sorted.length) - 1))]!; +} + +export function laterAnswerWaits(answers: readonly LaterAnswer[]): { count: number; queueWaitP50Ms: number | null; queueWaitMaxMs: number | null; + answerAfterPostP50Ms: number | null; answerAfterPostMaxMs: number | null } { + const queued = answers.flatMap(answer => answer.queuedAt === null ? [] : [answer.startedAt - answer.queuedAt]); + const posted = answers.flatMap(answer => answer.postedAt === null ? [] : [answer.startedAt - answer.postedAt]); + return { count: answers.length, queueWaitP50Ms: percentile(queued, 0.5), queueWaitMaxMs: queued.length ? Math.max(...queued) : null, + answerAfterPostP50Ms: percentile(posted, 0.5), answerAfterPostMaxMs: posted.length ? Math.max(...posted) : null }; +} diff --git a/tests/restart-chaos/grader.ts b/tests/restart-chaos/grader.ts index 8796bad647d..8c494c7070c 100644 --- a/tests/restart-chaos/grader.ts +++ b/tests/restart-chaos/grader.ts @@ -1,7 +1,7 @@ import { UI_RETRY_BLOCKED_BY_MODAL, UI_RETRY_BUTTON_MISSING, UI_RETRY_NO_REQUEST, type FaultReceipt, type OracleResult, type RuleId, type RuleResult, type TaskGrade, type TaskRecord, type TaskSample } from "./types.js"; import { g1Decision } from "./workloads.js"; import { gradeDiskFullRetry, isDiskFullFault } from "./disk-full-faults.js"; -import { enforcedUniversalFailures, gradeUniversal, type UniversalDbEvidence } from "./universal.js"; +import { enforcedUniversalFailures, gradeUniversal, type SlotFree, type UniversalBounds, type UniversalDbEvidence } from "./universal.js"; /** Pure R1–R6 grading over observer samples (plan §2). No I/O. */ @@ -192,7 +192,7 @@ export function gradeR4(record: TaskRecord): RuleResult { function namedWait(sample: TaskSample): boolean { const interruption = sample.interruption; - return Boolean(sample.executionBlocker || sample.scheduledRetry || sample.activeRecoveryAction || interruption?.retryWait + return Boolean(sample.executionBlocker || sample.scheduledRetry || sample.activeRecoveryAction || sample.waitingOnChildren || interruption?.retryWait || interruption?.state === "retry_scheduled" || interruption?.canRetry || (interruption && ["failed", "waiting_for_user"].includes(interruption.state))); } @@ -250,7 +250,7 @@ export function gradeR6(record: TaskRecord): RuleResult { } export function gradeTask(record: TaskRecord, faults: readonly FaultReceipt[], budgetMs: number, oracle: OracleResult, - universalDb?: UniversalDbEvidence): TaskGrade { + universalDb?: UniversalDbEvidence, queue: { bounds?: UniversalBounds; slotFree?: SlotFree } = {}): TaskGrade { const crossing = crossingFaults(record, faults); const rules: Record = { R1: gradeR1(record, budgetMs, faults), @@ -262,7 +262,7 @@ export function gradeTask(record: TaskRecord, faults: readonly FaultReceipt[], b }; const failed = (Object.entries(rules) as Array<[RuleId, RuleResult]>).find(([, rule]) => rule.verdict === "fail"); // Universal checks run on every task whatever its workload; only enforced ones fail it (universal.ts). - const universal = gradeUniversal(record, universalDb ? { db: universalDb } : {}); + const universal = gradeUniversal(record, { ...(universalDb ? { db: universalDb } : {}), ...queue }); const [universalFailure] = enforcedUniversalFailures(universal); const pass = !failed && oracle.verdict === "pass" && !universalFailure; let signature: string | null = null; diff --git a/tests/restart-chaos/journey-matrix.ts b/tests/restart-chaos/journey-matrix.ts index 3d434544ebb..f063c2497d8 100644 --- a/tests/restart-chaos/journey-matrix.ts +++ b/tests/restart-chaos/journey-matrix.ts @@ -87,7 +87,6 @@ export function applyJourneyOptions(options: WaveOptions, input: { matrix: boole options.tasks = matrixCells(options).length; } if (input.serverFrom !== undefined) { - if (options.env !== "local") throw new Error("--server-from runs only with --env local"); if (options.faults.includes("K14")) throw new Error("--server-from runs every server from one checkout; use --upgrade-from for K14"); options.serverFrom = path.resolve(input.serverFrom); } diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index 1c319e547b2..67b9c8e0325 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -27,9 +27,11 @@ import { DiskFullFaultSeam, isDiskFullFault, receiptsForIssue, withDiskFullCheck import { CREDENTIAL_FOR_ADAPTER, parseCommand, replayCommand, waveIdFor } from "./config.js"; import { assertControllerMatchesImage, assertProviderPackMismatch, defaultDaytonaImageCacheRoot, prepareDaytonaImageArtifacts } from "./daytona-image.js"; import { PLUGIN_BUILD_STAMP, pluginBuildState } from "./plugin-build.js"; +import { finishDaytonaCleanup } from "./daytona-cleanup.js"; import { DaytonaWaveSweeper, leaksForTask, unattributedLeaks, type DaytonaSweepResult } from "./daytona-sweep.js"; import { knownSecretValues, redactKnownSecrets, wakeEvidenceOnce, writeEvidenceBundle } from "./evidence.js"; import { gradeTask } from "./grader.js"; +import { loadWaveGradeInputs } from "./wave-grade-inputs.js"; import { releaseInstance } from "../runner-e2e/instance-retention.js"; import { checkPlanInvariants, readUniversalDbEvidence, summarizeViolations, type InvariantCheck } from "./invariants.js"; import { collectContentionEvidence, RunRowPressure } from "./contention.js"; @@ -140,7 +142,7 @@ async function prepareDaytona(plan: WavePlan, timeline: Timeline): Promise console.info(`[daytona-image] ${line}`) }); - const { localContentId } = await assertControllerMatchesImage(artifacts, repositoryRoot); + const { localContentId } = await assertControllerMatchesImage(artifacts, plan.options.serverFrom ?? repositoryRoot); timeline.add("daytona_image_matched", { image, contentId: artifacts.contentId, localContentId, sourceRevision: artifacts.sourceRevision, runnerdSha256: artifacts.runnerdSha256, providerPackDigest: artifacts.providerPackDigest, platformManifestDigest: artifacts.platformManifestDigest }); ensureDaytonaPlugin(timeline); @@ -789,6 +791,7 @@ async function main(): Promise { const universalByIssue = await readUniversalDbEvidence(evidenceDb, lab.companyId).catch(error => { timeline.add("universal_db_evidence_failed", { error: String(error) }); return null; }); const allRunMessages = new Map([...(universalByIssue?.values() ?? [])].flatMap(entry => [...entry.runMessages])); + const gradeInputs = await loadWaveGradeInputs(waveDir, plan.options.stagingMix); const costSummary = launched.some(observer => observer.plan.workload === "L1") ? await api.get>(`/api/companies/${lab.companyId}/costs/summary?period=all`) .then(summary => typeof summary.spendCents === "number" @@ -827,8 +830,9 @@ async function main(): Promise { observer.record.faultIds, parseRecoveryStepReceipts(await readFile(recoveryStepReceipts, "utf8").catch(() => "")), artifacts?.runs ?? []); // CR1–CR3: the task's own fault applied (CR2 killed a provider, CR3 stalled the plugin worker; class-r.ts). const toolOracle = withClassRCheck(stepOracle, observer.plan, faults); + Object.assign(observer.record, await gradeInputs.prepare(observer.plan.taskId, observer.record)); const grade = gradeTask(observer.record, faults, observer.definition.budgetMs, toolOracle, universalByIssue ? { - runMessages: allRunMessages, recoveryActions: universalByIssue.get(observer.record.issueId ?? "")?.recoveryActions ?? [] } : undefined); + runMessages: allRunMessages, recoveryActions: universalByIssue.get(observer.record.issueId ?? "")?.recoveryActions ?? [] } : undefined, gradeInputs.queue); let evidence: string | null = null; if (!grade.pass) { anyFailure = true; @@ -916,31 +920,12 @@ async function main(): Promise { await board?.close().catch(() => undefined); await closeJourneyApps(); try { if (sweeper) try { - const owned = sweeper as DaytonaWaveSweeper; - // An interrupted wave still reports what it leaked before cleanup. - sandboxSweep ??= await owned.sweep(0); - // Cleanup only through Paperclip; the harness never changes a sandbox. - const cleanup = await owned.cleanUp(); - daytonaCleanup = cleanup; - timeline.add("daytona_cleanup", cleanup); - console.info(`Daytona R6 sweep: ${sandboxSweep.liveSandboxes.length} live, ${sandboxSweep.archivedSandboxes.length} archived after release, ` - + `${sandboxSweep.orphanSandboxes.length} orphaned, ` - + `${sandboxSweep.heldLeases.length} held leases${sandboxSweep.error ? `, error ${sandboxSweep.error}` : ""}; ` - + `environment deleted: ${cleanup.environmentDeleted}${cleanup.environmentDeleteBlocked ? " (still blocked by a pending sandbox cleanup)" : ""}; still running after cleanup: ${cleanup.remainingSandboxes.join(", ") || "none"}`); - // A sandbox neither archived nor destroyed after cleanup is a leak for a person to act on. - if (cleanup.remainingSandboxes.length) { anyFailure = true; remoteCleanupPending = true; } - // A delete still blocked by pending sandbox cleanup after the settle window - // is a leak, not a harness error: fail the wave and keep the instance, whose - // database owns that cleanup work. - if (cleanup.environmentDeleteBlocked) { - anyFailure = true; - remoteCleanupPending = true; - timeline.add("daytona_environment_delete_blocked", { detail: cleanup.environmentDeleteBlocked }); - } - if (cleanup.errors.length) remoteCleanupPending = true; - if (cleanup.errors.length) harnessError ??= new Error(`Daytona cleanup incomplete: ${cleanup.errors.join("; ")}`); - await writeFile(path.join(waveDir, "sandbox-sweep.json"), `${JSON.stringify({ sweep: sandboxSweep, cleanup }, null, 2)}\n`, { mode: 0o600 }) - .catch(error => timeline.add("sandbox_sweep_write_failed", { error: String(error) })); + const outcome = await finishDaytonaCleanup({ sweeper: sweeper as DaytonaWaveSweeper, sweep: sandboxSweep, waveDir, timeline }); + sandboxSweep = outcome.sweep; + daytonaCleanup = outcome.cleanup; + if (outcome.failed) anyFailure = true; + if (outcome.remotePending) remoteCleanupPending = true; + if (outcome.error) harnessError ??= outcome.error; } catch (error) { harnessError ??= error; remoteCleanupPending = true; diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index c24c7e7dd25..ec339633307 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -52,6 +52,11 @@ export async function listWorkspaceFiles(root: string): Promise { export const QUEUED_TRIGGER_MAX_EMPTY_SAMPLES = 3; +export function bannerRestDue(since: number, now: number, raw = process.env.RESTART_CHAOS_MIX_BANNER_REST_SECONDS): boolean { + if (raw === undefined || raw === "" || !/^[0-9]+$/.test(raw)) return false; + return now - since >= Number(raw) * 1000; +} + function stillQueued(runs: readonly RunSample[]): boolean { return runs.some(run => run.status === "queued" && !run.startedAt) && !runs.some(run => run.startedAt || run.status === "running"); } @@ -207,6 +212,7 @@ export class TaskObserver { readonly record: TaskRecord; issue: Json | null = null; private restCandidateSince: number | null = null; + private bannerSince: number | null = null; private childrenSettledAt: number | null = null; private childStallCheckedAt: number | null = null; private stallSince: number | null = null; @@ -513,6 +519,10 @@ export class TaskObserver { this.restCandidateSince = null; return; } + if (isStalledSample(sample) && this.definition.collect.childDocuments && this.record.issueId) { + const children = await this.collectChildren(this.record.issueId).catch(() => null); + if (children?.length && childrenPending(children)) sample.waitingOnChildren = true; + } // A task waiting on the board (G1's approval, H1's paused company) is in a named wait, not a stall. this.stallSince = isStalledSample(sample) && !this.awaitingBoard() ? (this.stallSince ?? now) : null; if (this.stallSince !== null && now - this.stallSince > STALL_GIVE_UP_MS) { @@ -693,8 +703,13 @@ export class TaskObserver { if (status !== "done" && (!this.record.retry || faultSinceRetry) && sample.interruption?.canRetry) { if (this.plan.stagingMix?.retryDuringWave && holdsRetry(this.record, this.plan.stagingMix.holdMs, now)) { this.restCandidateSince = null; return; } if (!this.plan.stagingMix || this.plan.stagingMix.retryDuringWave) { await this.boardRetry(sample, now); return; } - this.restCandidateSince = null; - return; + this.bannerSince ??= now; + if (!bannerRestDue(this.bannerSince, now)) { + this.restCandidateSince = null; + return; + } + } else { + this.bannerSince = null; } this.record.restAt = this.restCandidateSince ?? now; this.record.finalStatus = status; diff --git a/tests/restart-chaos/release-swap-schedule.test.ts b/tests/restart-chaos/release-swap-schedule.test.ts index 9a028c52001..19266c280ba 100644 --- a/tests/restart-chaos/release-swap-schedule.test.ts +++ b/tests/restart-chaos/release-swap-schedule.test.ts @@ -32,6 +32,25 @@ describe("release swap options", () => { expect(run.due(1_500_000)).toBe(run.plan.faults[1]); }); + it("drives a staging-mix workload's slots on the swap clock instead of a storm", () => { + const { args, releaseSwap } = expandReleaseSwapArgs(["--env", "daytona", "--workload", "staging-mix", "--release-swap", "--release-swap-count", "2", + "--release-swap-spacing-minutes", "5", "--release-swap-overlap-seconds", "120", "--release-swap-settle-minutes", "2"]); + expect(releaseSwap).toEqual({ swaps: 2, spacingMs: 300_000, overlapMs: 120_000, settleMs: 120_000 }); + expect(args).toEqual(["--env", "daytona", "--workload", "staging-mix", "--mix-fault-spacing-minutes", "5", "--mix-minutes", "11", + "--faults", "K15", "--k15-overlap-seconds", "120", "--k15-fresh-home", "--ephemeral-server"]); + expect(() => expandReleaseSwapArgs(["--workload", "staging-mix", "--release-swap", "--mix-minutes", "30"])).toThrow(/--mix-minutes/); + const longest = ["--workload", "staging-mix", "--release-swap", "--release-swap-count", "6", "--release-swap-spacing-minutes"]; + expect(() => expandReleaseSwapArgs([...longest, "100"])).toThrow(/--release-swap-count 6 × --release-swap-spacing-minutes 100 needs a 601-minute staging-mix wave; the longest is 600/); + expect(expandReleaseSwapArgs([...longest, "99"]).args).toEqual(expect.arrayContaining(["--mix-minutes", "595"])); + const command = parseCommand(["--env", "local", "--storage", "s3", "--workload", "staging-mix", "--mix-tasks", "6", "--mix-runs", "4", "--mix-queue", "2", + "--mix-turn-minutes", "5", "--seed", "7", "--release-swap", "--release-swap-count", "2", "--release-swap-spacing-minutes", "5", + "--release-swap-overlap-seconds", "120", "--release-swap-settle-minutes", "2"], 0); + if (command.mode !== "wave") throw new Error("expected a wave"); + expect(command.options).toMatchObject({ faults: ["K15"], k15HoldMs: [120_000, 120_000], releaseSwap: { swaps: 2 }, + stagingMix: { faultSpacingMinutes: 5, minutes: 11, runs: 4, queue: 2, tasks: 6 } }); + expect(command.options.stormMinutes).toBeUndefined(); + }); + it("refuses flags the cell plans itself, stray shape flags, and an overlap the spacing cannot hold", () => { expect(() => expandReleaseSwapArgs(["--release-swap", "--faults", "K1"])).toThrow(/--faults/); expect(() => expandReleaseSwapArgs(["--release-swap-count", "2"])).toThrow(/add --release-swap/); diff --git a/tests/restart-chaos/release-swap-schedule.ts b/tests/restart-chaos/release-swap-schedule.ts index 179b94734e7..25fc20b7d0a 100644 --- a/tests/restart-chaos/release-swap-schedule.ts +++ b/tests/restart-chaos/release-swap-schedule.ts @@ -1,4 +1,5 @@ import type { StormFault, StormPlan } from "./types.js"; +import { MIX_MAX_MINUTES } from "./staging-mix.js"; export interface ReleaseSwapOptions { swaps: number; @@ -21,6 +22,8 @@ const FLAGS = { "release-swap-drain-minutes": { key: "drainMs", unit: 60_000, min: 1, max: 240 }, } as const; +const MIX_PLANNED = ["mix-fault-spacing-minutes", "mix-minutes"]; + const REPLACED = ["storm-minutes", "faults", "k15-overlap-seconds", "k15-hold-seconds", "replay", "crash-point", "matrix", "upgrade-from"]; function flagName(arg: string): string | null { @@ -73,8 +76,20 @@ export function expandReleaseSwapArgs(input: readonly string[]): { args: string[ } const envIndex = rest.findIndex(arg => arg === "--env" || arg.startsWith("--env=")); const env = envIndex < 0 ? "local" : rest[envIndex]!.includes("=") ? rest[envIndex]!.split("=")[1] : rest[envIndex + 1]; - const added = ["--storm-minutes", String(Math.min(240, Math.ceil((options.swaps * options.spacingMs) / 60_000))), "--faults", "K15", - "--k15-overlap-seconds", String(Math.round(options.overlapMs / 1000))]; + const mix = given.has("workload"); + if (mix) { + const planned = MIX_PLANNED.find(name => given.has(name)); + if (planned) throw new Error(`--release-swap plans the staging-mix swap slots; do not combine it with --${planned}`); + } + const spacingMinutes = Math.ceil(options.spacingMs / 60_000); + if (mix && options.swaps * spacingMinutes + 1 > MIX_MAX_MINUTES) { + throw new Error(`--release-swap-count ${options.swaps} × --release-swap-spacing-minutes ${spacingMinutes} needs a ${options.swaps * spacingMinutes + 1}-minute ` + + `staging-mix wave; the longest is ${MIX_MAX_MINUTES} minutes. Lower the swap count or the spacing.`); + } + const clock = mix + ? ["--mix-fault-spacing-minutes", String(spacingMinutes), "--mix-minutes", String(options.swaps * spacingMinutes + 1)] + : ["--storm-minutes", String(Math.min(240, Math.ceil((options.swaps * options.spacingMs) / 60_000)))]; + const added = [...clock, "--faults", "K15", "--k15-overlap-seconds", String(Math.round(options.overlapMs / 1000))]; if (env === "daytona") { if (!given.has("k15-fresh-home")) added.push("--k15-fresh-home"); if (!given.has("ephemeral-server")) added.push("--ephemeral-server"); diff --git a/tests/restart-chaos/release-swap-setup.ts b/tests/restart-chaos/release-swap-setup.ts index 4fb96461abd..6d8a7e65c74 100644 --- a/tests/restart-chaos/release-swap-setup.ts +++ b/tests/restart-chaos/release-swap-setup.ts @@ -30,19 +30,21 @@ export async function diskUsageKb(directory: string): Promise { } export async function startReleaseSwap(input: { options: ReleaseSwapOptions; env: EnvironmentKind; adapters: readonly AdapterId[]; lab: ChaosLab; - api: ChaosApi; db: Db; timeline: Timeline; temporaryRoot: string; freshHome: boolean; cancelled: () => boolean }): Promise { + api: ChaosApi; db: Db; timeline: Timeline; temporaryRoot: string; freshHome: boolean; cancelled: () => boolean }): Promise<{ driver: ReleaseSwapDriver; probesReady: Promise }> { const shape = input.env === "daytona" ? DAYTONA_PROBE_SHAPE : LOCAL_PROBE_SHAPE; const record = (kind: string, data: Record) => { input.timeline.add(kind, data); }; const agents = Object.fromEntries(input.adapters.flatMap(adapter => input.lab.agents[adapter] ? [[adapter, input.lab.agents[adapter]]] : [])); const probeAgents = await hireProbeAgents(input.api, input.lab.companyId, agents, shape.concurrency); - await waitForAgentsReady(input.api, probeAgents, input.timeline, input.env === "daytona" ? 10 * 60_000 : undefined, 1_000, input.cancelled); - record("release_swap_probe_agents", { agents: Object.fromEntries(Object.entries(probeAgents).map(([adapter, agent]) => [adapter, agent!.id])), - maxConcurrentRuns: shape.concurrency, prepIntervalMs: shape.prepIntervalMs, prepProbeCap: shape.prepProbeCap }); + const probesReady = waitForAgentsReady(input.api, probeAgents, input.timeline, input.env === "daytona" ? 10 * 60_000 : undefined, 1_000, input.cancelled).then(() => { + record("release_swap_probe_agents", { agents: Object.fromEntries(Object.entries(probeAgents).map(([adapter, agent]) => [adapter, agent!.id])), + maxConcurrentRuns: shape.concurrency, prepIntervalMs: shape.prepIntervalMs, prepProbeCap: shape.prepProbeCap }); + }); + probesReady.catch(() => undefined); const driver = new ReleaseSwapDriver({ options: input.options, companyId: input.lab.companyId, adapters: input.adapters, agents: input.lab.agents, probeAgents, prepIntervalMs: shape.prepIntervalMs, prepProbeCap: shape.prepProbeCap, reader: pgRunReader(input.db), record }); driver.plugins.push(homeCleanupPlugin({ temporaryRoot: input.temporaryRoot, freshHome: input.freshHome, remove: directory => rm(directory, { recursive: true, force: true }), diskUsageKb, record })); - return driver; + return { driver, probesReady }; } export function stormRunFor(plan: WavePlan, now: number): StormRun | ReleaseSwapRun | null { @@ -59,9 +61,9 @@ export class ReleaseSwapWave { supervisor: Pick; cancelled: () => boolean }): Promise { const options = input.plan.options; const db = createDb(input.databaseUrl); - const driver = await startReleaseSwap({ options: options.releaseSwap!, env: options.env, adapters: options.adapters, lab: input.lab, api: input.api, db, + const { driver, probesReady } = await startReleaseSwap({ options: options.releaseSwap!, env: options.env, adapters: options.adapters, lab: input.lab, api: input.api, db, timeline: input.timeline, temporaryRoot: input.supervisor.paths.temporaryRoot, freshHome: options.k15FreshHome === true, cancelled: input.cancelled }); - input.supervisor.swapHook = driver.hook; + input.supervisor.swapHook = async info => { await probesReady; return driver.hook(info); }; setOverlapProbe(() => driver.launchOpen); const wave = new ReleaseSwapWave(driver, db, input.plan, input.lab, input.api, input.timeline, input.supervisor, input.cancelled); driver.plugins.push({ name: "overlap-launch-deadlines", onPhase: async (_swap, info) => { if (info.phase === "stopped") wave.extendOverlapLaunches(info.at); } }); diff --git a/tests/restart-chaos/server-freeze.test.ts b/tests/restart-chaos/server-freeze.test.ts new file mode 100644 index 00000000000..963d8d692f6 --- /dev/null +++ b/tests/restart-chaos/server-freeze.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; +import { whileFrozen, type SendSignal } from "./server-freeze.js"; + +function signals(exited = false) { + const sent: Array<[number, string | number]> = []; + const send: SendSignal = (pid, signal) => { + if (exited && signal === (0 as unknown as NodeJS.Signals)) throw Object.assign(new Error("no such process"), { code: "ESRCH" }); + sent.push([pid, signal]); + }; + return { sent, send }; +} + +describe("freezing the incoming server while the outgoing one stops", () => { + it("resumes the incoming server after the freeze when the stop succeeds", async () => { + const { sent, send } = signals(); + const waits: number[] = []; + const frozen = await whileFrozen({ pid: 42, freezeMs: 150_000, send, wait: async ms => { waits.push(ms); } }, async () => "stopped"); + expect(frozen.result).toBe("stopped"); + expect(frozen.thawedAt).not.toBeNull(); + expect(sent.filter(([, signal]) => signal !== 0)).toEqual([[42, "SIGSTOP"], [42, "SIGCONT"]]); + expect(waits[0]).toBeGreaterThan(149_000); + }); + + it("resumes the incoming server at once when stopping the outgoing one fails", async () => { + const { sent, send } = signals(); + await expect(whileFrozen({ pid: 42, freezeMs: 150_000, send, wait: async () => undefined }, async () => { throw new Error("stop timed out"); })) + .rejects.toThrow(/stop timed out/); + expect(sent.filter(([, signal]) => signal !== 0)).toEqual([[42, "SIGSTOP"], [42, "SIGCONT"]]); + }); + + it("sends nothing after the incoming server exited, and nothing at all without a freeze", async () => { + const exited = signals(true); + await expect(whileFrozen({ pid: 42, freezeMs: 1_000, send: exited.send, wait: async () => undefined }, async () => { throw new Error("boom"); })).rejects.toThrow(); + expect(exited.sent).toEqual([[42, "SIGSTOP"]]); + const none = signals(); + expect(await whileFrozen({ pid: 42, freezeMs: null, send: none.send }, async () => 1)).toEqual({ result: 1, frozenAt: null, thawedAt: null }); + expect(none.sent).toEqual([]); + }); +}); diff --git a/tests/restart-chaos/server-freeze.ts b/tests/restart-chaos/server-freeze.ts new file mode 100644 index 00000000000..e2385363852 --- /dev/null +++ b/tests/restart-chaos/server-freeze.ts @@ -0,0 +1,31 @@ +import { setTimeout as delay } from "node:timers/promises"; + +export type SendSignal = (pid: number, signal: NodeJS.Signals) => void; + +const sendSignal: SendSignal = (pid, signal) => { process.kill(pid, signal); }; + +function alive(pid: number, send: SendSignal): boolean { + try { send(pid, 0 as unknown as NodeJS.Signals); return true; } catch { return false; } +} + +export async function whileFrozen(input: { pid: number | null | undefined; freezeMs: number | null; send?: SendSignal; wait?: (ms: number) => Promise }, + body: () => Promise): Promise<{ result: T; frozenAt: number | null; thawedAt: number | null }> { + const send = input.send ?? sendSignal; + const wait = input.wait ?? (ms => delay(ms)); + const frozenAt = input.freezeMs !== null && input.pid ? Date.now() : null; + if (frozenAt !== null) send(input.pid!, "SIGSTOP"); + let thawedAt: number | null = null; + const thaw = () => { + if (frozenAt === null || thawedAt !== null) return; + thawedAt = Date.now(); + if (alive(input.pid!, send)) send(input.pid!, "SIGCONT"); + }; + try { + const result = await body(); + if (frozenAt !== null) await wait(Math.max(0, frozenAt + input.freezeMs! - Date.now())); + thaw(); + return { result, frozenAt, thawedAt }; + } finally { + thaw(); + } +} diff --git a/tests/restart-chaos/staging-mix-retry.test.ts b/tests/restart-chaos/staging-mix-retry.test.ts index 8329327fd12..136314815c2 100644 --- a/tests/restart-chaos/staging-mix-retry.test.ts +++ b/tests/restart-chaos/staging-mix-retry.test.ts @@ -1,12 +1,12 @@ import { describe, expect, it } from "vitest"; import { parseCommand } from "./config.js"; import { - collectStagingMixArtifacts, completedAtAnchor, doneCommentsBeforeCompletion, doneMarker, expectedRoundHash, gradeStagingMix, planStagingMixWave, ROUND_SECONDS, + collectStagingMixArtifacts, completedAtAnchor, completionRunIds, doneCommentsBeforeCompletion, doneMarker, expectedRoundHash, gradeStagingMix, planStagingMixWave, ROUND_SECONDS, stagingMixDescription, type CompletionEvidence, type StagingMixTask, } from "./staging-mix.js"; import { - bannerSink, commentEvidence, evidenceCompletion, extraTurnLine, extraTurnsByReason, issueView, readAssignedWorkInvariant, retryAfterSettleStage, runsAfterDone, shouldClickRetryAfterSettle, - stagingMixHeadline, summarizeStagingMix, type RetryStageObserver, + bannerSink, commentEvidence, evidenceCompletion, extraTurnLine, extraTurnsByReason, followUpLine, followUpsAnsweredLater, issueView, laterAnswerHeadline, readAssignedWorkInvariant, + retryAfterSettleStage, runsAfterDone, shouldClickRetryAfterSettle, stagingMixHeadline, summarizeStagingMix, type RetryStageObserver, type WakeEvidenceFile, } from "./staging-mix-retry.js"; import type { OracleArtifacts, TaskPlan, TaskRecord, TaskSample } from "./types.js"; @@ -242,8 +242,8 @@ describe("staging-mix runs after the task completed", () => { expect(neverStarted.map(turn => turn.runId)).toEqual(["aa11bb22-dddd"]); expect(extraTurnLine(neverStarted[0]!)).toContain("never started"); expect(extraTurnsByReason(started)).toEqual({ issue_children_completed: 1, issue_recovery_action_restored: 1 }); - expect(runsAfterDone("t001", evidence, { at: null, source: null })).toEqual({ started: [], neverStarted: [] }); - expect(runsAfterDone("t001", null, { at: 1, source: "completed_at" })).toEqual({ started: [], neverStarted: [] }); + expect(runsAfterDone("t001", evidence, { at: null, source: null })).toEqual({ started: [], neverStarted: [], answeredFollowUp: [] }); + expect(runsAfterDone("t001", null, { at: 1, source: "completed_at" })).toEqual({ started: [], neverStarted: [], answeredFollowUp: [] }); }); it("does not count the run that completed the task as an extra turn (w-20261112T233630-s7201 t005)", () => { @@ -264,6 +264,79 @@ describe("staging-mix runs after the task completed", () => { }); }); +describe("a follow-up the running turn did not receive (w-20261113T052703-s7301 t010)", () => { + const nonce = "bd13d3bf"; + const comment = "fd278a3a-946b-44df-b62f-0e47c982aea1"; + const first = "0d18f0d9-0550-4f97-8926-0fcf49b43d19", second = "42acaf33-4c75-4d62-8bfe-07053a78cd9f"; + const t010 = (): WakeEvidenceFile => ({ + completedAt: 1_794_550_265_674, + runs: [{ runId: first, createdAt: 1_794_548_287_511, startedAt: 1_794_548_287_574, invocationSource: "assignment", triggerDetail: "system", + wakeupRequestId: "4816f3b7", acceptedDisposition: "done", acceptedAt: 1_794_549_206_395, contextSnapshot: { wakeReason: "issue_assigned", paperclipWake: { commentIds: [] } }, + }, + { runId: second, createdAt: 1_794_549_207_682, startedAt: 1_794_550_224_261, invocationSource: "automation", triggerDetail: "system", + wakeupRequestId: "452458ef", acceptedDisposition: "done", acceptedAt: 1_794_550_262_627, + contextSnapshot: { wakeReason: "issue_commented", wakeCommentIds: [comment], paperclipWake: { commentIds: [comment] } } }], + wakes: [{ id: "4816f3b7", reason: "issue_assigned", source: "assignment", triggerDetail: "system", requestedByActorType: "user", requestedByActorId: "local-board", createdAt: 1_794_548_287_511 }, + { id: "452458ef", reason: "issue_execution_promoted", source: "automation", triggerDetail: "system", requestedByActorType: "user", requestedByActorId: "local-board", + createdAt: 1_794_548_961_691, payload: { commentId: comment, _paperclipWakeContext: { wakeCommentIds: [comment] } } }], + }); + const posted = commentEvidence([ + { body: "Done. The follow-up requirement is applied: the `ledger` document now includes `FOLLOWUP-bd13d3bf`; the single `DONE-bd13d3bf` comment was already posted.", + authorAgentId: "a", createdByRunId: second, createdAt: "2026-11-13T06:11:05.788Z" }, + { body: "Task CHA-12 complete. No board comments arrived, so no extra lines were added.", authorAgentId: "a", createdByRunId: first, createdAt: "2026-11-13T05:53:27.608Z" }, + { body: doneMarker(nonce), authorAgentId: "a", createdByRunId: first, createdAt: "2026-11-13T05:53:19.468Z" }]); + const completionOnce = (evidence: WakeEvidenceFile) => gradeStagingMix(nonce, { turnMinutes: 15, rounds: 9, subtasks: 0, followUpAfterMs: 400_000, arriveAtMs: 0, holdMs: 0 }, + { issueStatus: "done", documents: { ledger: ledger(nonce, 9) }, agentComments: [], files: {}, commands: {}, workspacePath: null, + stagingMix: { followUp: null, followUpErrors: [], retries: [], comments: posted, completion: evidenceCompletion(evidence)! } }).checks.find(check => check.name === "completion_once")!; + const afterDone = (evidence: WakeEvidenceFile) => runsAfterDone("t010", evidence, completedAtAnchor(evidenceCompletion(evidence), posted, nonce)); + + it("counts the run that answered the unreceived comment as required, not as an extra turn or a duplicate completion", () => { + const evidence = t010(); + expect(afterDone(evidence).started).toEqual([]); + expect(afterDone(evidence).answeredFollowUp.map(turn => turn.runId)).toEqual([second]); + expect(completionOnce(evidence)).toMatchObject({ ok: true, detail: "runs=1" }); + const answered = followUpsAnsweredLater("t010", evidence); + expect(answered).toEqual([{ taskId: "t010", runId: second, commentIds: [comment], postedAt: 1_794_548_961_691, queuedAt: 1_794_549_207_682, startedAt: 1_794_550_224_261 }]); + expect(followUpLine(answered[0]!)).toBe("- t010: run 42acaf33 answered fd278a3a 1263 s after it was posted, after 1017 s queued"); + expect(laterAnswerHeadline(0, answered)).toEqual(["extra turns (a run started after completion with no undelivered input): 0", + "follow-up answered in a later run: 1, queued wait p50 1017 s / max 1017 s, posted to answered p50 1263 s / max 1263 s"]); + }); + + const withRun = (base: WakeEvidenceFile, runId: string, wake: { id: string; reason: string; payload?: unknown }, at: number): WakeEvidenceFile => { + const run2 = t010().runs![1]!; + return { ...base, runs: [...base.runs!, { ...run2, runId, createdAt: at, startedAt: at + 900, wakeupRequestId: wake.id, contextSnapshot: { wakeReason: wake.reason } }], + wakes: [...base.wakes!, { id: wake.id, reason: wake.reason, source: "automation", triggerDetail: "system", requestedByActorType: "system", requestedByActorId: null, + createdAt: at, payload: wake.payload ?? {} }] }; + }; + + it("treats the server's re-dispatch of an owed comment after a failed attempt as required", () => { + const base = t010(); + const failed = { ...base, runs: [base.runs![0]!, { ...base.runs![1]!, status: "failed", acceptedDisposition: null, acceptedAt: null }] }; + const evidence = withRun(failed, "9e9e9e9e-retry", { id: "retry-wake", reason: "process_lost_retry", payload: { _paperclipWakeContext: { wakeCommentIds: [comment] } } }, + 1_794_550_230_000); + expect(afterDone(evidence).started).toEqual([]); + expect(afterDone(evidence).answeredFollowUp.map(turn => turn.runId)).toEqual([second, "9e9e9e9e-retry"]); + expect(completionOnce(evidence)).toMatchObject({ ok: true, detail: "runs=1" }); + }); + + it("counts a run started after completion from a wake that carries no comment as an extra turn and a duplicate completion", () => { + const evidence = withRun(t010(), "7a7a7a7a-extra", { id: "recovery-wake", reason: "issue_recovery_action_restored" }, 1_794_550_300_000); + expect(afterDone(evidence).started.map(turn => turn.runId)).toEqual(["7a7a7a7a-extra"]); + expect(afterDone(evidence).answeredFollowUp.map(turn => turn.runId)).toEqual([second]); + expect(completionOnce(evidence)).toMatchObject({ ok: false, detail: "duplicate completion: runs 0d18f0d9, 7a7a7a7a each reported completion" }); + }); + + it("lists nothing for a comment wake the server cancelled as already delivered, and an extra run for the same comment is still extra", () => { + const base = t010(); + const cancelled: WakeEvidenceFile = { ...base, runs: [base.runs![0]!] }; + expect(followUpsAnsweredLater("t010", cancelled)).toEqual([]); + expect(afterDone(cancelled)).toEqual({ started: [], neverStarted: [], answeredFollowUp: [] }); + const replayed = withRun(cancelled, "5b5b5b5b-replay", { id: "replay-wake", reason: "issue_recovery_action_restored" }, 1_794_550_300_000); + expect(afterDone(replayed).started.map(turn => turn.runId)).toEqual(["5b5b5b5b-replay"]); + expect(completionOnce(replayed).ok).toBe(false); + }); +}); + describe("staging-mix completion from accepted results", () => { const nonce = "abcd1234"; const completion = (runs: CompletionEvidence["runs"]): CompletionEvidence => ({ completedAt: 10_000, runs }); @@ -283,6 +356,31 @@ describe("staging-mix completion from accepted results", () => { stagingMix: { followUp: null, followUpErrors: [], retries: [], comments: posted, completion: twice } }).checks.find(check => check.name === "completion_once")!.ok).toBe(false); }); + type Row = { runId: string; at: number; disposition: string | null; followUp?: boolean }; + const first = (disposition: string | null): Row => ({ runId: "first", at: 1_000, disposition }); + const followUp = (at: number, disposition: string | null): Row => ({ runId: `follow-up-${at}`, at, disposition, followUp: true }); + const extra = (at: number): Row => ({ runId: `extra-${at}`, at, disposition: "done" }); + it.each<{ name: string; rows: Row[]; ok: boolean; runs: string[] }>([ + { name: "first completion by the first run", rows: [first("done")], ok: true, runs: ["first"] }, + { name: "first completion by a follow-up run after a yielded first run", rows: [first("yielded"), followUp(2_000, "done")], ok: true, runs: ["follow-up-2000"] }, + { name: "first completion by the retry of a follow-up run that failed before its turn", rows: [first("yielded"), followUp(2_000, null), followUp(3_000, "done")], + ok: true, runs: ["follow-up-3000"] }, + { name: "a later completion that answers new input", rows: [first("done"), followUp(2_000, "done")], ok: true, runs: ["first"] }, + { name: "a later completion that answers nothing", rows: [first("done"), extra(2_000)], ok: false, runs: ["first", "extra-2000"] }, + { name: "a follow-up run completes first, then a run that answers nothing completes again", rows: [first("yielded"), followUp(2_000, "done"), extra(3_000)], + ok: false, runs: ["follow-up-2000", "extra-3000"] }, + { name: "a later follow-up completion, then a run that answers nothing", rows: [first("done"), followUp(2_000, "done"), extra(3_000)], + ok: false, runs: ["first", "extra-3000"] }, + ])("completion_once truth table: $name", ({ rows, ok, runs }) => { + const evidence = completion(rows.map(row => ({ runId: row.runId, startedAt: row.at - 500, acceptedDisposition: row.disposition, + acceptedAt: row.disposition ? row.at : null, answersFollowUp: row.followUp ?? false }))); + expect(completionRunIds([], nonce, evidence)).toEqual(runs); + const check = gradeStagingMix(nonce, { turnMinutes: 5, rounds: 4, subtasks: 0, followUpAfterMs: null, arriveAtMs: 0, holdMs: 0 }, + { issueStatus: "done", documents: { ledger: ledger(nonce, 4) }, agentComments: [], files: {}, commands: {}, workspacePath: null, + stagingMix: { followUp: null, followUpErrors: [], retries: [], comments: [], completion: evidence } }).checks.find(entry => entry.name === "completion_once")!; + expect(check.ok).toBe(ok); + }); + it("reports a DONE comment from a run whose accepted result was not done as its own finding", () => { const runs = completion([{ runId: "r1", startedAt: 1, acceptedDisposition: "blocked", acceptedAt: 5_000 }, { runId: "r2", startedAt: 6_000, acceptedDisposition: "done", acceptedAt: 9_000 }]); expect(doneCommentsBeforeCompletion([{ body: doneMarker(nonce), runId: "r1", createdAt: 4_900 }, { body: doneMarker(nonce), runId: "r2", createdAt: 8_900 }], nonce, runs)) diff --git a/tests/restart-chaos/staging-mix-retry.ts b/tests/restart-chaos/staging-mix-retry.ts index cea5753242c..1b52d73dd5f 100644 --- a/tests/restart-chaos/staging-mix-retry.ts +++ b/tests/restart-chaos/staging-mix-retry.ts @@ -6,6 +6,7 @@ import { bannerShownMs, completedAtAnchor, doneCommentsBeforeCompletion, DUPLICATE_CHECKS, firstDoneCommentAt, noteRetryBanner, type CompletionEvidence, } from "./staging-mix.js"; import type { InterruptionSample, OracleResult, TaskPlan, TaskRecord, TaskSample } from "./types.js"; +import { laterAnswers, laterAnswerWaits, type LaterAnswer } from "./follow-up-delivery.js"; type Json = Record; @@ -191,10 +192,13 @@ export interface ExtraTurn { export interface WakeEvidenceFile { completedAt?: number | null; runs?: Array<{ runId: string | null; createdAt: number | null; startedAt?: number | null; invocationSource: string | null; triggerDetail: string | null; - wakeupRequestId: string | null; acceptedDisposition?: string | null; acceptedAt?: number | null }>; - wakes?: Array<{ id: string | null; reason: string | null; source: string | null; triggerDetail: string | null; requestedByActorType: string | null; requestedByActorId: string | null }>; + wakeupRequestId: string | null; acceptedDisposition?: string | null; acceptedAt?: number | null; finishedAt?: number | null; contextSnapshot?: unknown; status?: string | null }>; + wakes?: Array<{ id: string | null; reason: string | null; source: string | null; triggerDetail: string | null; requestedByActorType: string | null; requestedByActorId: string | null; + createdAt?: number | null; payload?: unknown }>; } +export interface FollowUpAnsweredLater extends LaterAnswer { taskId: string } + export type CommentRow = { body?: unknown; authorAgentId?: unknown; createdByRunId?: unknown; createdAt?: unknown }; export function commentEvidence(rows: readonly CommentRow[] | null): Array<{ body: string; runId: string | null; createdAt: number | null }> { @@ -207,14 +211,16 @@ export function commentEvidence(rows: readonly CommentRow[] | null): Array<{ bod export function evidenceCompletion(evidence: WakeEvidenceFile | null, issueCompletedAt?: unknown): CompletionEvidence | null { const fromIssue = typeof issueCompletedAt === "string" && Number.isFinite(Date.parse(issueCompletedAt)) ? Date.parse(issueCompletedAt) : null; if (!evidence && fromIssue === null) return null; + const answers = laterAnswers(evidence); return { completedAt: evidence?.completedAt ?? fromIssue, runs: (evidence?.runs ?? []).map(run => ({ runId: run.runId, startedAt: run.startedAt ?? null, - acceptedDisposition: run.acceptedDisposition ?? null, acceptedAt: run.acceptedAt ?? null })) }; + acceptedDisposition: run.acceptedDisposition ?? null, acceptedAt: run.acceptedAt ?? null, answersFollowUp: Boolean(run.runId && answers.has(run.runId)) })) }; } export function runsAfterDone(taskId: string, evidence: WakeEvidenceFile | null, anchor: { at: number | null; source: string | null }, doneCommentAt: number | null = null): - { started: ExtraTurn[]; neverStarted: ExtraTurn[] } { + { started: ExtraTurn[]; neverStarted: ExtraTurn[]; answeredFollowUp: ExtraTurn[] } { const doneAt = anchor.at; - if (!evidence || doneAt === null) return { started: [], neverStarted: [] }; + if (!evidence || doneAt === null) return { started: [], neverStarted: [], answeredFollowUp: [] }; + const answers = laterAnswers(evidence); const wakes = new Map((evidence.wakes ?? []).map(wake => [wake.id, wake])); const turn = (run: NonNullable[number]): ExtraTurn => { const wake = run.wakeupRequestId ? wakes.get(run.wakeupRequestId) : undefined; @@ -223,12 +229,33 @@ export function runsAfterDone(taskId: string, evidence: WakeEvidenceFile | null, actorType: wake?.requestedByActorType ?? null, actorId: wake?.requestedByActorId ?? null }; }; const runs = evidence.runs ?? []; + const afterDone = runs.filter(run => run.startedAt != null && run.startedAt > doneAt); + const answered = (run: (typeof runs)[number]) => Boolean(run.runId && answers.has(run.runId)); return { - started: runs.filter(run => run.startedAt != null && run.startedAt > doneAt).map(turn), + started: afterDone.filter(run => !answered(run)).map(turn), neverStarted: runs.filter(run => run.startedAt == null && run.createdAt !== null && run.createdAt > doneAt).map(turn), + answeredFollowUp: afterDone.filter(answered).map(turn), }; } +export function followUpsAnsweredLater(taskId: string, evidence: WakeEvidenceFile | null): FollowUpAnsweredLater[] { + return [...laterAnswers(evidence).values()].map(answer => ({ taskId, ...answer })); +} + +const seconds = (value: number | null) => value === null ? "?" : `${Math.round(value / 1000)} s`; + +export function followUpLine(answer: FollowUpAnsweredLater): string { + return `- ${answer.taskId}: run ${answer.runId.slice(0, 8)} answered ${answer.commentIds.map(id => id.slice(0, 8)).join(", ")} ` + + `${seconds(answer.postedAt === null ? null : answer.startedAt - answer.postedAt)} after it was posted, after ${seconds(answer.queuedAt === null ? null : answer.startedAt - answer.queuedAt)} queued`; +} + +export function laterAnswerHeadline(extraTurns: number, answers: readonly FollowUpAnsweredLater[]): string[] { + const waits = laterAnswerWaits(answers); + return [`extra turns (a run started after completion with no undelivered input): ${extraTurns}`, + `follow-up answered in a later run: ${waits.count}${waits.count ? `, queued wait p50 ${seconds(waits.queueWaitP50Ms)} / max ${seconds(waits.queueWaitMaxMs)}, ` + + `posted to answered p50 ${seconds(waits.answerAfterPostP50Ms)} / max ${seconds(waits.answerAfterPostMaxMs)}` : ""}`]; +} + const iso = (at: number | null) => at === null ? "?" : new Date(at).toISOString(); export function extraTurnLine(turn: ExtraTurn): string { @@ -264,6 +291,7 @@ export async function writeStagingMixResults(input: { waveDir: string; api: Pick const tasks: ResultTask[] = []; const extraTurns: ExtraTurn[] = []; const queuedAfterDone: ExtraTurn[] = []; + const answeredLater: FollowUpAnsweredLater[] = []; const doneBeforeCompletion: Array<{ taskId: string; runId: string; disposition: string }> = []; for (const task of input.tasks) { const issue = task.record.issueId ? await input.api.get(`/api/issues/${task.record.issueId}`, 10_000).catch(() => null) : null; @@ -276,6 +304,7 @@ export async function writeStagingMixResults(input: { waveDir: string; api: Pick doneBeforeCompletion.push(...doneCommentsBeforeCompletion(posted, task.plan.nonce, completion).map(entry => ({ taskId: task.plan.taskId, ...entry }))); extraTurns.push(...afterDone.started); queuedAfterDone.push(...afterDone.neverStarted); + answeredLater.push(...followUpsAnsweredLater(task.plan.taskId, evidence)); const grade = await readFile(path.join(input.waveDir, `${task.plan.taskId}.grade.json`), "utf8") .then(text => JSON.parse(text) as { grade?: { T?: OracleResult } }).catch(() => null); tasks.push({ taskId: task.plan.taskId, issueId: task.record.issueId, record: task.record, finalStatus: issue?.status ?? task.record.finalStatus, @@ -283,16 +312,19 @@ export async function writeStagingMixResults(input: { waveDir: string; api: Pick } const result = summarizeStagingMix(tasks, extraClicks, await readAssignedWorkInvariant(input.api, input.companyId)); await writeFile(path.join(input.waveDir, "staging-mix-results.json"), `${JSON.stringify({ schema: "paperclip.restart-chaos.staging-mix/v1", - headline: stagingMixHeadline(result), ...result, extraTurns, extraTurnsByReason: extraTurnsByReason(extraTurns), queuedAfterDone, doneBeforeCompletion }, null, 2)}\n`, { mode: 0o600 }); + headline: [...stagingMixHeadline(result), ...laterAnswerHeadline(extraTurns.length, answeredLater)], ...result, extraTurns, extraTurnsByReason: extraTurnsByReason(extraTurns), + queuedAfterDone, doneBeforeCompletion, followUpsAnsweredLater: answeredLater, followUpWaits: laterAnswerWaits(answeredLater) }, null, 2)}\n`, { mode: 0o600 }); const invariant = result.invariant.violationCount === null ? `unreadable${result.invariant.error ? ` (${result.invariant.error})` : ""}` : `${result.invariant.violationCount} violations, ${result.invariant.agentsWithStalledSlots} agents with stalled slots, ${result.invariant.openTasks} open tasks`; - await writeFile(path.join(input.waveDir, "staging-mix-summary.md"), [...stagingMixHeadline(result), `Assigned-work invariant: ${invariant}`, "", + await writeFile(path.join(input.waveDir, "staging-mix-summary.md"), [...stagingMixHeadline(result), ...laterAnswerHeadline(extraTurns.length, answeredLater), + `Assigned-work invariant: ${invariant}`, "", ...result.duplicatedSideEffects.map(entry => `- ${entry.taskId}: duplicated ${entry.checks.join(", ")}`), ...result.evidenceIncomplete.map(entry => `- ${entry.taskId}: evidence incomplete for ${entry.checks.join(", ")}`), ...result.bannerShown.map(entry => `- ${entry.taskId}: Retry banner from ${entry.firstShownAt} for ${Math.round(entry.shownMs / 1000)} s` + `${entry.errorCode ? ` (${entry.errorCode})` : ""}${entry.completedAnyway ? ", completed without a click" : ""}`), ...(extraTurns.length ? ["", `Runs started after the task completed: ${extraTurns.length}`, ...extraTurns.map(extraTurnLine)] : []), ...(queuedAfterDone.length ? ["", `Runs queued after the task completed, never started: ${queuedAfterDone.length}`, ...queuedAfterDone.map(extraTurnLine)] : []), + ...(answeredLater.length ? ["", `Follow-ups the running turn did not receive, answered in a later run: ${answeredLater.length}`, ...answeredLater.map(followUpLine)] : []), ...(doneBeforeCompletion.length ? ["", `DONE comment posted before the task completed: ${doneBeforeCompletion.length}`, ...doneBeforeCompletion.map(entry => `- ${entry.taskId}: run ${entry.runId.slice(0, 8)} posted DONE but its accepted result was ${entry.disposition}`)] : []), ""].join("\n"), { mode: 0o600 }); return result; diff --git a/tests/restart-chaos/staging-mix.test.ts b/tests/restart-chaos/staging-mix.test.ts index 7dcfd30df49..c6c82ef5cbc 100644 --- a/tests/restart-chaos/staging-mix.test.ts +++ b/tests/restart-chaos/staging-mix.test.ts @@ -4,7 +4,7 @@ import { parseCommand } from "./config.js"; import { agentRunCaps, assignAdapters, childLine, childNonce, doneMarker, expectedRoundHash, followUpMarker, gradeStagingMix, holdsRetry, planArrivals, arrivalEndMs, followUpRequestId, planStagingMixWave, plannedTaskCount, posixCksum, retryDue, roundCommand, ROUND_SECONDS, setOverlapProbe, stagingMixDescription, - stagingMixBudgetMs, stagingMixStep, SUBTASK_ROUNDS, verifyLedger, type StagingMixTask, + slotTimesMs, stagingMixBudgetMs, stagingMixStep, SUBTASK_ROUNDS, verifyLedger, type StagingMixTask, } from "./staging-mix.js"; import { descendantsOf, parseMemInfo, parsePs, parseVmStat, StagingMixRun, stagingMixSeedOptions, type ResourceSample } from "./staging-mix-run.js"; import type { OracleArtifacts, TaskRecord, TaskSample } from "./types.js"; @@ -115,6 +115,14 @@ describe("staging-mix options", () => { }); describe("staging-mix planning", () => { + it("moves the first fault slot early without adding a slot (release swap: 2 swaps 4 minutes apart)", () => { + expect(slotTimesMs({ minutes: 9, faultSpacingMinutes: 4 }, undefined)).toEqual([240_000, 480_000]); + expect(slotTimesMs({ minutes: 9, faultSpacingMinutes: 4 }, "30")).toEqual([30_000, 270_000]); + expect(slotTimesMs({ minutes: 29, faultSpacingMinutes: 14 }, "60")).toEqual([60_000, 900_000]); + expect(slotTimesMs({ minutes: 28, faultSpacingMinutes: 14 }, "60")).toEqual([60_000]); + expect(slotTimesMs({ minutes: 3, faultSpacingMinutes: 4 }, "30")).toEqual([]); + }); + it("interleaves adapters so every prefix keeps the ratio", () => { const adapters = assignAdapters([["opencode", 60], ["codex", 25], ["claude", 15]], 100); const count = (slice: string[], adapter: string) => slice.filter(entry => entry === adapter).length; diff --git a/tests/restart-chaos/staging-mix.ts b/tests/restart-chaos/staging-mix.ts index b84536c645a..649e9667fb0 100644 --- a/tests/restart-chaos/staging-mix.ts +++ b/tests/restart-chaos/staging-mix.ts @@ -5,6 +5,7 @@ import type { AdapterId, FaultKind, OracleArtifacts, OracleResult, TaskFaultKind import type { WorkloadAgents, WorkloadDefinition } from "./workloads.js"; export const STAGING_MIX = "staging-mix"; +export const MIX_MAX_MINUTES = 600; export const ROUND_TICKS = 45; export const ROUND_SECONDS = ROUND_TICKS * 2; export const SUBTASKS_PER_PARENT = 2; @@ -60,7 +61,7 @@ export interface StagingMixRecord { export interface CompletionEvidence { completedAt: number | null; - runs: Array<{ runId: string | null; startedAt: number | null; acceptedDisposition: string | null; acceptedAt: number | null }>; + runs: Array<{ runId: string | null; startedAt: number | null; acceptedDisposition: string | null; acceptedAt: number | null; answersFollowUp?: boolean }>; } export interface RetryBanner { firstShownAt: number; lastShownAt: number; shownMs: number; visibleSince: number | null; errorCode: string | null } @@ -145,7 +146,7 @@ export function applyStagingMix(options: WaveOptions, values: MixArgs, environme throw new Error(`--workload ${STAGING_MIX} takes restart faults only`); } const turnMinutes = parseRange(String(values["mix-turn-minutes"] ?? "15-45"), "mix-turn-minutes", 1, 60); - const minutes = whole(values["mix-minutes"], "mix-minutes", 240, 1, 600); + const minutes = whole(values["mix-minutes"], "mix-minutes", 240, 1, MIX_MAX_MINUTES); const faultSpacingMinutes = whole(values["mix-fault-spacing-minutes"], "mix-fault-spacing-minutes", 40, 1, 600); const arrivalCutoffMinutes = whole(values["mix-arrival-cutoff-minutes"], "mix-arrival-cutoff-minutes", turnMinutes[1], 0, 600); const runCapDefault = whole(environment.RESTART_CHAOS_MIX_RUN_CAP, "RESTART_CHAOS_MIX_RUN_CAP", 30, 1, 200); @@ -220,10 +221,12 @@ function meanCycleMs(mix: Pick): number { return ((mix.turnMinutes[0] + mix.turnMinutes[1]) / 2 + 5) * 60_000; } -export function slotTimesMs(mix: Pick): number[] { - const times: number[] = []; - for (let at = mix.faultSpacingMinutes * 60_000; at < mix.minutes * 60_000; at += mix.faultSpacingMinutes * 60_000) times.push(at); - return times; +export function slotTimesMs(mix: Pick, + firstSlotRaw = process.env.RESTART_CHAOS_MIX_FIRST_SLOT_SECONDS): number[] { + const spacing = mix.faultSpacingMinutes * 60_000; + const first = firstSlotRaw && /^[0-9]+$/.test(firstSlotRaw) ? Math.min(Number(firstSlotRaw) * 1000, spacing) : spacing; + const count = Math.max(0, Math.ceil((mix.minutes * 60_000 - spacing) / spacing)); + return Array.from({ length: count }, (_, index) => first + index * spacing); } export function maxSlotHoldMs(options: Pick): number { @@ -409,8 +412,16 @@ export function completionRunIds(comments: readonly CommentEvidence[], nonce: st const summaries = comments.filter(comment => anchor !== null && typeof comment.createdAt === "number" && comment.createdAt >= anchor && comment.body.includes(doneMarker(nonce)) && comment.runId).map(comment => comment.runId!); if (hasDispositions(completion)) { + const byId = new Map(completion.runs.flatMap(run => run.runId ? [[run.runId, run] as const] : [])); + const reportedAt = (runId: string) => { + const run = byId.get(runId); + if (run?.acceptedDisposition === "done" && run.acceptedAt !== null) return run.acceptedAt; + const posted = comments.filter(comment => comment.runId === runId && typeof comment.createdAt === "number").map(comment => comment.createdAt!); + return posted.length ? Math.min(...posted) : run?.startedAt ?? Number.POSITIVE_INFINITY; + }; const accepted = completion.runs.filter(run => run.acceptedDisposition === "done" && run.runId).map(run => run.runId!); - return [...new Set([...accepted, ...summaries])]; + const ordered = [...new Set([...accepted, ...summaries])].sort((a, b) => reportedAt(a) - reportedAt(b)); + return ordered.filter((runId, index) => index === 0 || !byId.get(runId)?.answersFollowUp); } return [...new Set([...comments.filter(comment => isDoneComment(comment.body, nonce) && comment.runId).map(comment => comment.runId!), ...summaries])]; } diff --git a/tests/restart-chaos/supervisor.ts b/tests/restart-chaos/supervisor.ts index 99c6205132b..61062e49628 100644 --- a/tests/restart-chaos/supervisor.ts +++ b/tests/restart-chaos/supervisor.ts @@ -42,6 +42,7 @@ import { isMultiServerFault, runMultiServerFault, type MultiServerKind, type Mul import { generationHome, nextRedeployIdentity, redeployEnvironment, redeployIdentity, redeployNodeArgs, redeployReceipt, type RedeployIdentity } from "./redeploy.js"; import { startEdgeProxy, type EdgeProxy } from "./edge-proxy.js"; import { ephemeralServerEnvironment } from "./ephemeral-server.js"; +import { whileFrozen } from "./server-freeze.js"; import type { SwapPhase, SwapPhaseInfo } from "./release-swap-driver.js"; import { CLASS_R_PRE_FAULT_DEADLINE_MS, classRRedeploys, isClassRFault, runPluginStall, withReplacementPluginStall, type ClassRHooks } from "./class-r.js"; @@ -74,6 +75,12 @@ export function crashLoopShouldFire(target: (typeof CRASH_LOOP_TARGETS)[number], return phase === "ready"; } +export function overlapKillAfterMs(raw: string | undefined, overlapMs: number, name = "RESTART_CHAOS_K15_KILL_OLD_AFTER_SECONDS"): number | null { + if (raw === undefined || raw === "") return null; + if (!/^[0-9]+$/.test(raw)) throw new Error(`${name} must be a whole number of seconds`); + return Math.min(Number(raw) * 1000, overlapMs); +} + /** * Agents reach the server at PAPERCLIP_API_URL. An explicit value wins over the URL the server * derives from its public URL (a developer's .paperclip/.env or shell can name a tailnet origin @@ -176,6 +183,7 @@ export class ChaosSupervisor { deployShape: { overlap: boolean; freshHome: boolean } = { overlap: false, freshHome: false }; ephemeralServer = false; swapHook: ((info: SwapPhaseInfo) => Promise) | null = null; + private rollingRedeploys = 0; /** `--k15-fresh-home`: the instance home and config of the serving generation (generation 0 uses `paths`). */ private serverHome: { home: string; configPath: string }; /** `--k15-overlap-seconds`: the edge on the wave's port, and the serving generation's own port behind it. */ @@ -822,10 +830,29 @@ export class ChaosSupervisor { this.edge.target = this.backendPort; this.timeline.add("k15_traffic_switched", { faultId: receipt.faultId, fromPort, toPort: this.backendPort, overlapMs }); await swap("switched", next); - for (const until = Date.now() + overlapMs; Date.now() < until && !this.shuttingDown;) await delay(Math.min(1_000, until - Date.now())); - await swap("stopping", next); await this.stopper(previous, "SIGTERM"); - receipt.stoppedAt = Date.now(); - this.timeline.add("fault_server_down", { faultId: receipt.faultId, previousPid: previous.pid, overlapMs }); + this.rollingRedeploys += 1; + const killSwaps = process.env.RESTART_CHAOS_K15_KILL_OLD_SWAPS?.split(",").map(Number) ?? null; + const killAfterMs = killSwaps && !killSwaps.includes(this.rollingRedeploys) ? null + : overlapKillAfterMs(process.env.RESTART_CHAOS_K15_KILL_OLD_AFTER_SECONDS, overlapMs); + for (const until = Date.now() + (killAfterMs ?? overlapMs); Date.now() < until && !this.shuttingDown;) await delay(Math.min(1_000, until - Date.now())); + await swap("stopping", next); + const freezeSwaps = process.env.RESTART_CHAOS_K15_FREEZE_SWAPS?.split(",").map(Number) ?? null; + const freezeMs = freezeSwaps && !freezeSwaps.includes(this.rollingRedeploys) ? null + : overlapKillAfterMs(process.env.RESTART_CHAOS_K15_FREEZE_NEW_SECONDS, 600_000, "RESTART_CHAOS_K15_FREEZE_NEW_SECONDS"); + if (freezeMs !== null && next.pid) this.timeline.add("k15_new_server_frozen", { faultId: receipt.faultId, pid: next.pid, forMs: freezeMs }); + const frozen = await whileFrozen({ pid: next.pid, freezeMs }, async () => { + if (killAfterMs === null) await this.stopper(previous, "SIGTERM"); + else { + this.timeline.add("k15_old_server_killed", { faultId: receipt.faultId, previousPid: previous.pid, afterMs: killAfterMs }); + await this.stopper.crashForRestart(previous); + } + receipt.stoppedAt = Date.now(); + this.timeline.add("fault_server_down", { faultId: receipt.faultId, previousPid: previous.pid, overlapMs }); + }); + if (frozen.frozenAt !== null) { + this.timeline.add("k15_new_server_thawed", { faultId: receipt.faultId, pid: next.pid, frozenMs: frozen.thawedAt! - frozen.frozenAt, + afterOldExitMs: frozen.thawedAt! - receipt.stoppedAt! }); + } await swap("stopped", next); await this.redeployEvidence?.(receipt.faultId, "stopped"); } diff --git a/tests/restart-chaos/ttft-baseline.ts b/tests/restart-chaos/ttft-baseline.ts index 7ac5281f74f..aa81bd08bf6 100644 --- a/tests/restart-chaos/ttft-baseline.ts +++ b/tests/restart-chaos/ttft-baseline.ts @@ -331,8 +331,8 @@ async function main() { const cleanup = await owned.cleanUp(); timeline.add("daytona_cleanup", cleanup); await writeFile(path.join(outDir, "sandbox-cleanup.json"), `${JSON.stringify(cleanup, null, 2)}\n`, { mode: 0o600 }); - if (cleanup.remainingSandboxes.length || cleanup.errors.length || cleanup.environmentDeleteBlocked || !cleanup.environmentDeleted) { - process.stderr.write(`Daytona cleanup incomplete: remaining ${cleanup.remainingSandboxes.join(", ") || "none"}; ` + if (cleanup.remainingSandboxes.length || cleanup.undeletedSandboxes.length || cleanup.errors.length || cleanup.environmentDeleteBlocked || !cleanup.environmentDeleted) { + process.stderr.write(`Daytona cleanup incomplete: remaining ${cleanup.remainingSandboxes.join(", ") || "none"}; left after deletion ${cleanup.undeletedSandboxes.join(", ") || "none"}; ` + `environment deleted ${cleanup.environmentDeleted}${cleanup.environmentDeleteBlocked ? ` (blocked: ${String(cleanup.environmentDeleteBlocked)})` : ""}; ${cleanup.errors.join("; ")}\n`); process.exitCode = 1; } diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index 1e528be26cb..a9cb8c61f2a 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -294,6 +294,7 @@ export interface TaskSample { recoveryOwner?: string | null; executionBlocker?: boolean; scheduledRetry?: boolean; + waitingOnChildren?: boolean; runs?: RunSample[]; } diff --git a/tests/restart-chaos/universal.test.ts b/tests/restart-chaos/universal.test.ts index 84ad5eb3093..ee85e466646 100644 --- a/tests/restart-chaos/universal.test.ts +++ b/tests/restart-chaos/universal.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from "vitest"; import { gradeTask } from "./grader.js"; import type { OracleArtifacts, RunSample, TaskRecord, TaskSample } from "./types.js"; -import { bareCauseMessage, DEFAULT_UNIVERSAL_BOUNDS as BOUNDS, diagnosableFailureCause, gradeU1, gradeU2, gradeU3, gradeU4, gradeU5, gradeUniversal, universalBounds } from "./universal.js"; +import { bareCauseMessage, declaredQueueBoundMs, DEFAULT_UNIVERSAL_BOUNDS as BOUNDS, diagnosableFailureCause, gradeU1, gradeU2, gradeU3, gradeU4, gradeU5, gradeUniversal, + universalBounds } from "./universal.js"; +import { slotFreeFrom } from "./agent-slots.js"; import { workloadDefinitions } from "./workloads.js"; const T0 = Date.parse("2026-11-09T12:00:00Z"); @@ -48,6 +50,33 @@ describe("U1 liveness", () => { expect(result.detail).toMatch(/^no_owner_no_action .* in todo with r1 queued/); }); + it("lets a run wait in a declared queue while its agent's slots are full, and still fails one queued with a free slot (w-20261113T052703-s7301 t030)", () => { + const queuedThenStarted = [...polls(0, 486 * SEC, at => sample(at, "todo", [run("r1", "queued", 0)])), + ...polls(489 * SEC, 600 * SEC, at => sample(at, "in_progress", [run("r1", "running", 486 * SEC, null, { lastEvidenceAt: iso(at) })]))]; + const declared = { ...BOUNDS, declaredQueuedMs: declaredQueueBoundMs({ runs: 20, queue: 20, turnMinutes: [12, 22] }) }; + expect(declared.declaredQueuedMs).toBe(27 * 60 * SEC); + expect(gradeU1(record(queuedThenStarted), BOUNDS)).toMatchObject({ verdict: "fail", detail: expect.stringMatching(/^no_owner_no_action .* in todo with r1 queued/) }); + expect(gradeU1(record(queuedThenStarted), declared, () => false).verdict).toBe("pass"); + expect(gradeU1(record(queuedThenStarted), declared, () => null).verdict).toBe("pass"); + const free = gradeU1(record(queuedThenStarted), declared, () => true); + expect(free).toMatchObject({ verdict: "fail", detail: expect.stringMatching(/r1 queued \d+s \(\d+s with a free slot, declared queue bound 1620s\)/) }); + const pastDeclared = polls(0, 31 * 60 * SEC, at => sample(at, "todo", [run("r1", "queued", 0)])); + expect(gradeU1(record(pastDeclared, { restAt: null, finalStatus: null, timedOut: true }), declared, () => false).verdict).toBe("fail"); + expect(declaredQueueBoundMs({ runs: 20, queue: 0, turnMinutes: [12, 22] })).toBeUndefined(); + expect(declaredQueueBoundMs(undefined)).toBeUndefined(); + }); + + it("reads free slots from the agent's own runs and its run cap", () => { + const slotFree = slotFreeFrom({ agents: [{ agentId: "a1", maxConcurrentRuns: 2 }, { agentId: "a2", maxConcurrentRuns: null }], + runs: [{ runId: "q", agentId: "a1", startedAt: 500, finishedAt: null }, { runId: "x", agentId: "a1", startedAt: 0, finishedAt: 300 }, + { runId: "y", agentId: "a1", startedAt: 100, finishedAt: null }, { runId: "z", agentId: "a2", startedAt: 0, finishedAt: null }] }); + expect(slotFree("q", 200)).toBe(false); + expect(slotFree("q", 400)).toBe(true); + expect(slotFree("z", 10)).toBeNull(); + expect(slotFree("unknown", 10)).toBeNull(); + expect(slotFreeFrom(null)("q", 1)).toBeNull(); + }); + it("treats a visible Retry or a waiting question as actionable", () => { const failedRun = run("r1", "failed", 0, 5 * SEC, { errorCode: "process_lost" }); const withRetry = polls(0, 10 * 60 * SEC, at => sample(at, "in_progress", [failedRun], { interruption: retryable })); diff --git a/tests/restart-chaos/universal.ts b/tests/restart-chaos/universal.ts index 8306aa081c1..0fdf88ac07f 100644 --- a/tests/restart-chaos/universal.ts +++ b/tests/restart-chaos/universal.ts @@ -32,6 +32,14 @@ export interface UniversalBounds { /** U4: longest a failure or Retry notice may remain after a later run succeeded. */ staleNoticeMs: number; maxRuns: number; + declaredQueuedMs?: number; +} + +export type SlotFree = (runId: string, at: number) => boolean | null; + +export function declaredQueueBoundMs(mix: { runs: number; queue: number; turnMinutes: readonly [number, number] } | undefined): number | undefined { + if (!mix || mix.queue <= 0) return undefined; + return Math.ceil(mix.queue / Math.max(1, mix.runs)) * (mix.turnMinutes[1] + 5) * 60_000; } /** @@ -140,6 +148,7 @@ export function liveOwner(sample: TaskSample, queuedFor: ReadonlyMap { +export function gradeU1(record: TaskRecord, bounds: UniversalBounds, slotFree: SlotFree = () => null): Omit { if (!record.samples.some(ready)) return { verdict: "n/a", detail: "no ready sample" }; // Ready-server time each run has spent queued (outages do not count). + const totalQueued = new Map(); + const freeQueued = new Map(); const queuedFor = new Map(); let orphanMs = 0, worstOrphan = 0, orphanFrom: number | null = null, lastAt: number | null = null; let worstQueued = 0, worstQueuedRun = ""; @@ -172,9 +183,14 @@ export function gradeU1(record: TaskRecord, bounds: UniversalBounds): Omit worstQueued) { worstQueued = waited; worstQueuedRun = run.id; } + if (bounds.declaredQueuedMs === undefined || run.status !== "queued") { queuedFor.set(run.id, waited); continue; } + const free = (freeQueued.get(run.id) ?? 0) + (slotFree(run.id, sample.at) === true ? step : 0); + freeQueued.set(run.id, free); + queuedFor.set(run.id, waited > bounds.declaredQueuedMs ? Number.POSITIVE_INFINITY : free); } } timerFor = retryTimer(sample) && sample.interruption?.retryWait ? timerFor + (lastAt !== null ? sample.at - lastAt : 0) : 0; @@ -189,7 +205,8 @@ export function gradeU1(record: TaskRecord, bounds: UniversalBounds): Omit bounds.orphanMs) { const queued = (sample.runs ?? []).filter(run => run.status === "queued" || run.status === "scheduled_retry"); return { verdict: "fail", metrics: { ownerlessMs: orphanMs, queuedMs: worstQueued, retryTimerMs: worstTimer }, detail: `no_owner_no_action ${seconds(orphanMs)} in ${sample.issueStatus ?? "unknown"}` - + (queued.length ? ` with ${queued.map(run => `${run.id} queued ${seconds(queuedFor.get(run.id) ?? 0)}`).join(",")}` : "") + + (queued.length ? ` with ${queued.map(run => `${run.id} queued ${seconds(totalQueued.get(run.id) ?? 0)}` + + `${freeQueued.has(run.id) ? ` (${seconds(freeQueued.get(run.id)!)} with a free slot, declared queue bound ${seconds(bounds.declaredQueuedMs!)})` : ""}`).join(",")}` : "") + (timerFor ? ` after a retry wait of ${seconds(timerFor)}${sample.interruption?.retryWait ? " (accepted Retry waiting on the system)" : ""}` : "") + ` (interruption ${sample.interruption?.state ?? "none"})` }; } @@ -304,11 +321,11 @@ export function gradeU5(record: TaskRecord, bounds: UniversalBounds): Omit } = {}): UniversalGrade { + enforced?: ReadonlySet; slotFree?: SlotFree } = {}): UniversalGrade { const bounds = options.bounds ?? universalBounds(); const enforced = options.enforced ?? ENFORCED_UNIVERSAL; const results = { - U1: gradeU1(record, bounds), + U1: gradeU1(record, bounds, options.slotFree), U2: gradeU2(record, bounds), U3: gradeU3(record, options.db), U4: gradeU4(record, bounds), diff --git a/tests/restart-chaos/wake-evidence.test.ts b/tests/restart-chaos/wake-evidence.test.ts index 2ff7ba29eba..438a0836b55 100644 --- a/tests/restart-chaos/wake-evidence.test.ts +++ b/tests/restart-chaos/wake-evidence.test.ts @@ -129,13 +129,17 @@ describe.skipIf(!support.supported)("wake evidence against real PostgreSQL", () it("writes one file per task with the wake row, context snapshot, status changes and serving controller", async () => { const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), wakeId = randomUUID(), runId = randomUUID(), bootId = randomUUID(); await db.insert(companies).values({ id: companyId, name: "Wake evidence", issuePrefix: companyId.slice(0, 8) }); - await db.insert(agents).values({ id: agentId, companyId, name: "Agent", adapterType: "process", status: "active" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Agent", adapterType: "process", status: "active", + runtimeConfig: { heartbeat: { enabled: false, maxConcurrentRuns: 3 } } }); + const wideAgentId = randomUUID(); + await db.insert(agents).values({ id: wideAgentId, companyId, name: "Wide agent", adapterType: "process", status: "active", + runtimeConfig: { heartbeat: { enabled: false, maxConcurrentRuns: 100 } } }); await db.insert(issues).values({ id: issueId, companyId, title: "Task" }); await db.insert(controllerInstances).values({ bootId, processSpace: "space-wake", pid: 4242, leaseExpiresAt: new Date(Date.now() + 60_000) }); await db.insert(agentWakeupRequests).values({ id: wakeId, companyId, agentId, source: "on_demand", reason: "retry", triggerDetail: "manual", requestedByActorType: "user", requestedByActorId: "u1", idempotencyKey: "idem-1", payload: { issueId }, runId }); await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, status: "succeeded", invocationSource: "on_demand", wakeupRequestId: wakeId, - contextSnapshot: { wakeReason: "retry", taskId: issueId }, controllerBootId: bootId }); + contextSnapshot: { wakeReason: "retry", taskId: issueId }, controllerBootId: bootId, startedAt: new Date(1_800_000_000_000), finishedAt: new Date(1_800_000_060_000) }); await db.insert(activityLog).values({ companyId, actorType: "agent", actorId: agentId, action: "issue.updated", entityType: "issue", entityId: issueId, details: { status: "done", _previous: { status: "in_progress" } } }); const waveDir = await mkdtemp(path.join(os.tmpdir(), "wake-evidence-")); @@ -146,6 +150,9 @@ describe.skipIf(!support.supported)("wake evidence against real PostgreSQL", () const evidence = JSON.parse(await readFile(path.join(waveDir, "wake-evidence", "t01.json"), "utf8")); expect(evidence.runs[0]).toMatchObject({ runId, invocationSource: "on_demand", wakeupRequestId: wakeId, contextSnapshot: { wakeReason: "retry" }, owner: { pid: 4242, processSpace: "space-wake" } }); + const slots = JSON.parse(await readFile(path.join(waveDir, "wake-evidence", "agent-slots.json"), "utf8")); + expect(slots.agents).toEqual(expect.arrayContaining([{ agentId, maxConcurrentRuns: 3 }, { agentId: wideAgentId, maxConcurrentRuns: 50 }])); + expect(slots.runs).toEqual([{ runId, agentId, startedAt: 1_800_000_000_000, finishedAt: 1_800_000_060_000 }]); expect(evidence.wakes[0]).toMatchObject({ id: wakeId, reason: "retry", source: "on_demand", triggerDetail: "manual", requestedByActorType: "user", requestedByActorId: "[REDACTED]", idempotencyKey: "idem-1", runId }); expect(evidence.wakes[0].servingAtCreate.map((server: { pid: number }) => server.pid)).toEqual([4242]); diff --git a/tests/restart-chaos/wake-evidence.ts b/tests/restart-chaos/wake-evidence.ts index 7bf4eaf0a51..4cb126718a1 100644 --- a/tests/restart-chaos/wake-evidence.ts +++ b/tests/restart-chaos/wake-evidence.ts @@ -4,7 +4,10 @@ import type { ServerProcessWindow } from "./multi-server.js"; import { unshiftJson, unshiftMs, type AgingShift } from "./time-faults.js"; import { ISSUE_STATUSES } from "../../packages/shared/src/constants.js"; import type { CompletionEvidence } from "./staging-mix.js"; -import { createDb, type Db } from "../../packages/db/src/index.js"; +import { laterAnswers, type DeliveryRun, type DeliveryWake } from "./follow-up-delivery.js"; +import { AGENT_SLOTS_FILE, agentSlotsFromRows } from "./agent-slots.js"; +import { agents, createDb, type Db } from "../../packages/db/src/index.js"; +import { maxConcurrentRunsSql } from "../../server/src/services/heartbeat/timer-policy.js"; type Row = Record; @@ -60,6 +63,7 @@ export interface WakeEvidenceInput { controllers: readonly ControllerRow[]; windows: readonly ServerProcessWindow[]; issues?: readonly Row[]; + agents?: readonly Row[]; } export function wakeEvidenceForIssue(issueId: string, input: WakeEvidenceInput) { @@ -75,8 +79,7 @@ export function wakeEvidenceForIssue(issueId: string, input: WakeEvidenceInput) const owner = controllerByBoot.get(String(run.controllerBootId ?? "")); return { runId: text(run.id), status: text(run.status), invocationSource: text(run.invocationSource), triggerDetail: text(run.triggerDetail), wakeupRequestId: text(run.wakeupRequestId), createdAt: ms(run.createdAt), startedAt: ms(run.startedAt), finishedAt: ms(run.finishedAt), - acceptedDisposition: text(run.acceptedDisposition), acceptedAt: ms(run.acceptedAt), contextSnapshot: run.contextSnapshot ?? null, - owner: owner ? { pid: owner.pid, bootId: owner.bootId, epoch: owner.epoch, processSpace: owner.processSpace } : null, + acceptedDisposition: text(run.acceptedDisposition), acceptedAt: ms(run.acceptedAt), contextSnapshot: run.contextSnapshot ?? null, owner: owner ? { pid: owner.pid, bootId: owner.bootId, epoch: owner.epoch, processSpace: owner.processSpace } : null, servingAtCreate: servingAt(ms(run.createdAt), controllers, input.windows) }; }), wakes: wakes.map(wake => ({ id: text(wake.id), reason: text(wake.reason), source: text(wake.source), triggerDetail: text(wake.triggerDetail), @@ -89,10 +92,10 @@ export function wakeEvidenceForIssue(issueId: string, input: WakeEvidenceInput) }; } -async function readRows(db: Pick, companyId: string, shifts: readonly AgingShift[]): Promise> { +async function readRows(db: Pick, companyId: string, shifts: readonly AgingShift[]): Promise> { const client = db.$client; const runs = await client.unsafe( - `select r.id::text as id, r.issue_id::text as "issueId", r.status, r.invocation_source as "invocationSource", r.trigger_detail as "triggerDetail", + `select r.id::text as id, r.agent_id::text as "agentId", r.issue_id::text as "issueId", r.status, r.invocation_source as "invocationSource", r.trigger_detail as "triggerDetail", r.wakeup_request_id::text as "wakeupRequestId", r.context_snapshot as "contextSnapshot", r.controller_boot_id::text as "controllerBootId", floor(extract(epoch from r.created_at) * 1000) as "createdAt", floor(extract(epoch from r.started_at) * 1000) as "startedAt", floor(extract(epoch from r.finished_at) * 1000) as "finishedAt", accepted.disposition as "acceptedDisposition", @@ -117,6 +120,8 @@ async function readRows(db: Pick, companyId: string, shifts: read floor(extract(epoch from created_at) * 1000) as "createdAt" from activity_log where company_id::text = $1 and entity_type = 'issue' and action like 'issue.%' and details ? 'status' order by created_at`, [companyId]) as Row[]; + const agentRows = (await db.select({ id: agents.id, companyId: agents.companyId, maxConcurrentRuns: maxConcurrentRunsSql() }).from(agents)) + .filter(agent => agent.companyId === companyId) as Row[]; const controllers = (await client.unsafe( `select boot_id as "bootId", epoch, process_space as "processSpace", pid, floor(extract(epoch from started_at) * 1000) as "startedAt", floor(extract(epoch from coalesce(stopped_at, fenced_at)) * 1000) as "endedAt" from controller_instances order by epoch`) as Row[]) @@ -125,6 +130,7 @@ async function readRows(db: Pick, companyId: string, shifts: read return { runs: unshiftRows(runs, ["createdAt", "startedAt", "finishedAt", "acceptedAt"], ["contextSnapshot"], shifts), issues: unshiftRows(issues, ["completedAt"], [], shifts), + agents: agentRows, wakes: unshiftRows(wakes, ["createdAt", "requestedAt", "claimedAt", "finishedAt"], ["payload"], shifts), statusChanges: unshiftRows(statusChanges, ["createdAt"], [], shifts), controllers: controllers.map(controller => ({ ...controller, startedAt: unshiftMs(controller.startedAt, shifts), @@ -132,7 +138,7 @@ async function readRows(db: Pick, companyId: string, shifts: read }; } -export async function captureWakeEvidence(input: { db: Pick; companyId: string; waveDir: string; +export async function captureWakeEvidence(input: { db: Pick; companyId: string; waveDir: string; observers: ReadonlyArray | null; windows: readonly ServerProcessWindow[]; shifts?: readonly AgingShift[]; redact: (text: string) => string }): Promise { const rows = await readRows(input.db, input.companyId, input.shifts ?? []); @@ -150,6 +156,7 @@ export async function captureWakeEvidence(input: { db: Pick; comp written += 1; } await writeFile(path.join(directory, "servers.json"), `${JSON.stringify({ controllers: rows.controllers, harnessWindows: input.windows }, null, 2)}\n`, { mode: 0o600 }); + await writeFile(path.join(directory, AGENT_SLOTS_FILE), `${JSON.stringify(agentSlotsFromRows(rows.agents ?? [], rows.runs), null, 2)}\n`, { mode: 0o600 }); return written; } @@ -158,15 +165,17 @@ export interface EvidenceObserver { record: { issueId: string | null; stagingMix?: { completion?: CompletionEvidence } }; } -export function completionEvidence(evidence: ReturnType): CompletionEvidence { - return { completedAt: evidence.completedAt, runs: evidence.runs.map(run => ({ runId: run.runId, startedAt: run.startedAt, - acceptedDisposition: run.acceptedDisposition, acceptedAt: run.acceptedAt })) }; +export function completionEvidence(evidence: { completedAt: number | null; wakes?: readonly DeliveryWake[]; + runs: ReadonlyArray }): CompletionEvidence { + const answers = laterAnswers(evidence); + return { completedAt: evidence.completedAt, runs: evidence.runs.map(run => ({ runId: run.runId, startedAt: run.startedAt ?? null, + acceptedDisposition: run.acceptedDisposition, acceptedAt: run.acceptedAt, answersFollowUp: Boolean(run.runId && answers.has(run.runId)) })) }; } export function wakeEvidenceOnce(input: { databaseUrl: () => string; companyId: string; waveDir: string; observers: () => ReadonlyArray; windows: readonly ServerProcessWindow[]; shifts: readonly AgingShift[]; redact: (text: string) => string; note: (kind: string, data: Record) => void; - open?: (url: string) => Pick }): () => Promise { + open?: (url: string) => Pick }): () => Promise { let done: Promise | null = null; return () => done ??= (async () => { let observers: ReturnType | null = null; diff --git a/tests/restart-chaos/warm-shutdown.ts b/tests/restart-chaos/warm-shutdown.ts index 3006b68c68c..cc8bc6e087d 100644 --- a/tests/restart-chaos/warm-shutdown.ts +++ b/tests/restart-chaos/warm-shutdown.ts @@ -2,6 +2,7 @@ export interface WarmShutdownCleanup { environmentDeleted: boolean; environmentDeleteBlocked: string | null; remainingSandboxes: string[]; + undeletedSandboxes?: string[]; } export interface WarmShutdownCheck { @@ -34,12 +35,13 @@ export function gradeWarmShutdown(input: { cleanup: WarmShutdownCleanup | null; if (input.cleanup?.environmentDeleteBlocked) problems.push(`environment delete blocked: ${input.cleanup.environmentDeleteBlocked}`); else if (input.cleanup && !input.cleanup.environmentDeleted) problems.push("environment was not deleted"); if (input.cleanup?.remainingSandboxes.length) problems.push(`sandboxes still running after cleanup: ${input.cleanup.remainingSandboxes.join(", ")}`); + if (input.cleanup?.undeletedSandboxes?.length) problems.push(`wave sandboxes left after the harness deleted them: ${input.cleanup.undeletedSandboxes.join(", ")}`); for (const line of shutdownCloseFailures) problems.push(`warm session did not close: ${line}`); return { verdict: problems.length ? "fail" : "pass", environmentDeleted: input.cleanup?.environmentDeleted ?? null, environmentDeleteBlocked: input.cleanup?.environmentDeleteBlocked ?? null, - leakedSandboxes: input.cleanup?.remainingSandboxes ?? [], + leakedSandboxes: [...new Set([...input.cleanup?.remainingSandboxes ?? [], ...input.cleanup?.undeletedSandboxes ?? []])], shutdownCloseFailures, idleCloseFailures, problems, diff --git a/tests/restart-chaos/wave-grade-inputs.ts b/tests/restart-chaos/wave-grade-inputs.ts new file mode 100644 index 00000000000..a2d7295e557 --- /dev/null +++ b/tests/restart-chaos/wave-grade-inputs.ts @@ -0,0 +1,28 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { AGENT_SLOTS_FILE, slotFreeFrom, type AgentSlots } from "./agent-slots.js"; +import { withChildWaits } from "./child-wait.js"; +import type { WakeEvidenceFile } from "./staging-mix-retry.js"; +import type { StagingMixOptions } from "./staging-mix.js"; +import type { TaskRecord } from "./types.js"; +import { declaredQueueBoundMs, universalBounds, type SlotFree, type UniversalBounds } from "./universal.js"; + +export interface WaveGradeInputs { + queue: { bounds: UniversalBounds; slotFree: SlotFree }; + slotEvidence: boolean; + prepare(taskId: string, record: TaskRecord): Promise; +} + +const readJson = async (file: string): Promise => readFile(file, "utf8").then(text => JSON.parse(text) as T).catch(() => null); + +export async function loadWaveGradeInputs(waveDir: string, mix: Pick | undefined, + env: NodeJS.ProcessEnv = process.env): Promise { + const slots = await readJson(path.join(waveDir, "wake-evidence", AGENT_SLOTS_FILE)); + const declaredQueuedMs = declaredQueueBoundMs(mix); + const bounds = { ...universalBounds(env), ...(declaredQueuedMs === undefined ? {} : { declaredQueuedMs }) }; + return { + queue: { bounds, slotFree: slotFreeFrom(slots) }, + slotEvidence: slots !== null, + prepare: async (taskId, record) => withChildWaits(record, await readJson(path.join(waveDir, "wake-evidence", `${taskId}.json`))), + }; +}