diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md
index b2f2a52ccbf..820eddc7225 100644
--- a/tests/restart-chaos/README.md
+++ b/tests/restart-chaos/README.md
@@ -557,6 +557,52 @@ node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona
--daytona-warm --daytona-cpu 2 --adapters opencode,codex,claude --release-swap-short --concurrency 12 --tasks 200 --seed 7110
```
+With `--workload staging-mix`, the swaps become the mix's fault slots (`--mix-fault-spacing-minutes` and `--mix-minutes`
+come from the swap count and spacing), so the staging-mix tasks, follow-ups and hands-off results run under the swap
+driver's probes and grading. Probe agents finish their verification while the first tasks run; the first swap waits for them.
+`--server-from
` also runs on Daytona: every server comes from that checkout while the sandboxes keep
+`--daytona-image`'s Runner and provider pack, as a deploy newer or older than its sandbox image does.
+
+### Staging twin cells
+
+Two local servers, volume-free, real Daytona sandboxes, two overlap swaps. `$IMG` must match the checkout's Runner
+(`--server-from`'s checkout when given). Run each through the wave queue (`~/.cache/paperclip/chaos-wave-acquire.sh`).
+
+```sh
+set -a; source ~/.secrets; set +a
+# Twin cell, variant G (graceful swaps): 6 tasks, 6 runs, 4-minute turns, 2 swaps 4 minutes apart, 120 s overlap. 18-23 minutes.
+node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona --daytona-image $IMG --daytona-warm --daytona-cpu 1 --storage s3 \
+ --workload staging-mix --mix-tasks 6 --mix-runs 6 --mix-queue 0 --mix-ratio opencode=34,codex=33,claude=33 --mix-turn-minutes 4 \
+ --mix-hold-minutes 2 --mix-arrival-cutoff-minutes 0 --release-swap --release-swap-count 2 --release-swap-spacing-minutes 4 \
+ --release-swap-overlap-seconds 120 --release-swap-settle-minutes 1 --seed 7105
+# Variant G2: the same command; the outgoing server is SIGKILLed 120 s into each overlap.
+RESTART_CHAOS_K15_KILL_OLD_AFTER_SECONDS=120 node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts ...
+# Variant G3: G2, and the incoming server is also frozen (SIGSTOP) for 150 s while the outgoing one stops, so it reaches the Runners late.
+RESTART_CHAOS_K15_KILL_OLD_AFTER_SECONDS=120 RESTART_CHAOS_K15_FREEZE_NEW_SECONDS=150 node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts ...
+# Loaded twin L1: 48 tasks, 20 runs plus 20 queued, 12-22 minute turns, 2 swaps 13 minutes apart, swap 1 graceful, swap 2 G3. About 78 minutes.
+RESTART_CHAOS_DAYTONA_VCPU_BUDGET=48 RESTART_CHAOS_K15_KILL_OLD_SWAPS=2 RESTART_CHAOS_K15_KILL_OLD_AFTER_SECONDS=120 \
+RESTART_CHAOS_K15_FREEZE_SWAPS=2 RESTART_CHAOS_K15_FREEZE_NEW_SECONDS=150 RESTART_CHAOS_MIX_BANNER_REST_SECONDS=240 \
+node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona --daytona-image $IMG --daytona-warm --daytona-cpu 1 --storage s3 \
+ --workload staging-mix --mix-tasks 48 --mix-runs 20 --mix-queue 20 --mix-ratio opencode=34,codex=33,claude=33 --mix-turn-minutes 12-22 \
+ --mix-hold-minutes 2 --mix-arrival-cutoff-minutes 0 --release-swap --release-swap-count 2 --release-swap-spacing-minutes 13 \
+ --release-swap-overlap-seconds 120 --release-swap-settle-minutes 1 --seed 7301
+```
+
+| Knob | Effect |
+| --- | --- |
+| `RESTART_CHAOS_K15_KILL_OLD_AFTER_SECONDS` | SIGKILL the outgoing server this many seconds into the overlap (at most the overlap) instead of a SIGTERM at its end |
+| `RESTART_CHAOS_K15_KILL_OLD_SWAPS` | comma-separated swap numbers (1-based) the kill applies to; unset means every swap |
+| `RESTART_CHAOS_K15_FREEZE_NEW_SECONDS` | SIGSTOP the incoming server for this long while the outgoing one stops, then SIGCONT |
+| `RESTART_CHAOS_K15_FREEZE_SWAPS` | swap numbers the freeze applies to; unset means every swap |
+| `RESTART_CHAOS_MIX_BANNER_REST_SECONDS` | a hands-off staging-mix task whose Retry banner has stood this long rests (it is still reported as a banner) instead of waiting out its budget |
+| `RESTART_CHAOS_MIX_FIRST_SLOT_SECONDS` | place the first staging-mix fault slot this early (at most the slot spacing); the slot count stays the same, so a release swap never gets an extra swap |
+
+A run counts as a success only when `staging-mix-summary.md` shows every started task completed hands-off
+(`completed hands-off / started` equal), `Retry banner shown: 0`, `Retry worked / Retry clicked` equal, `extra turns: 0`
+and `duplicated side effects: 0`, the release-swap headline shows every task completed without human action, and the
+Daytona cleanup left no sandbox (`left after deletion: none`). The `follow-up answered in a later run` line is not a
+failure; read its waits. In L1 the queued tasks wait for a slot by design; U1 allows that wait (see "Universal checks").
+
`--release-swap` runs the full default (6 swaps, 40 minutes apart). `--release-swap-short` runs 3 swaps 12 minutes
apart with the full 510 s overlap, so two servers run for most of each cycle. Either takes the `--release-swap-*`
options below, which override the preset. Workload sizing (turn length, concurrency) comes from the workload flags.
@@ -1040,12 +1086,20 @@ disk), and `--concurrency × --daytona-cpu` must stay within 40 vCPU
org limit with room for verification probes: a 10 vCPU org fails creates with
HTTP 400 "Total CPU limit exceeded", so use 8 there and `--daytona-cpu 2`.
-**Fleet safety.** The harness never stops, archives or deletes a Daytona
-sandbox. Its Daytona client is a frozen wrapper that exposes only a
-label-filtered `list`. `assertHarnessMayMutate` refuses `paperclip-create-*`
-names (every plugin-created sandbox has one) and `fleet-sbx-*` snapshots, so a
-future mutating path cannot act on fleet sandboxes either. Cleanup goes only
-through Paperclip, by deleting the wave's own environment.
+**Fleet safety.** The Daytona org is shared with the live Cloud fleet, and
+every plugin-created sandbox, the fleet's included, is named
+`paperclip-create-*`. The name therefore says nothing about who owns a
+sandbox; the labels do. Each wave's isolated instance mints a fresh company
+and environment UUID, and the Daytona plugin labels every sandbox it creates
+with both. Cleanup first deletes the wave's own environment through Paperclip.
+Then it deletes every sandbox that still carries both of this wave's labels
+(archived setup sandboxes included, since Daytona does not delete archived
+sandboxes on its own), and nothing else: the listing is filtered by both labels
+on the server and checked again on every row, and `assertHarnessMayMutate`
+refuses any sandbox without both labels and any `fleet-sbx-*` snapshot. The
+only delete is `DELETE /sandbox/` for a row that passed those checks. A
+wave sandbox still there after the settle window fails the wave
+(`undeletedSandboxes`, and the warm-shutdown verdict).
**Image matching.** The controller needs the image's own Linux amd64
`/usr/local/bin/paperclip-runnerd` (`PAPERCLIP_RUNNER_REMOTE_BINARY_PATH`) and
@@ -1552,12 +1606,16 @@ click instead, and its issue reads also update the banner record. Both paths use
So no task gets two clicks. `--mix-retry-during-wave` restores the old behaviour: Retries
clicked during the wave, held until an overlap window as follow-ups are (`staging_mix_retry_in_overlap`).
-**Results.** `staging-mix-results.json` and the first lines of `SUMMARY.md` give four lines, in this order:
+**Results.** `staging-mix-results.json` and the first lines of `SUMMARY.md` give six lines, in this order:
1. `completed hands-off / started`.
2. `duplicated side effects: N tasks`, followed by `; evidence incomplete: M tasks` when a check could not read its evidence (such a check is never counted as a duplicate).
3. `Retry banner shown: N tasks`.
4. `Retry worked / Retry clicked`.
+5. `extra turns`: runs that started after the task completed with no undelivered input.
+6. `follow-up answered in a later run: n`, with the answering run's queued wait (p50 / max) and the time from the comment
+ to the answering run's start (p50 / max). These runs are required, not extra turns; the waits are shown because a long
+ one is a product concern (a person waits for an answer behind a full queue).
A task counts as completed hands-off when all three hold, read from fresh data at the end of the wave:
@@ -1575,6 +1633,18 @@ The summary also lists, by start time, every run that started after the task com
the task completed" (`extraTurns`, with counts by wake reason in `extraTurnsByReason`). A run queued after completion
that never started is listed on its own (`queuedAfterDone`).
+The grader takes comment delivery from the server's own record and does not decide it again. A comment that the server
+judged delivered to an earlier run never becomes a run: its wake is cancelled or coalesced. A run that the server
+started from a wake that carries a person's comment posted after the task's first run started (`wakeCommentIds`, the
+wake payload's `commentId`) is a required run. This includes the retry of a failed attempt for the same comment. OpenCode,
+for example, takes no input mid-turn, so a comment posted during its turn is answered by the next run. Such runs are
+listed under "Follow-ups the running turn did not receive, answered in a later run" (`followUpsAnsweredLater`,
+`followUpWaits`). An extra turn is a run that started after completion from a wake that carries no comment. The
+server's delivery rule (`task_history.read`, cited evidence, child-result reads) is covered by the server's own tests.
+
+`completion_once` stays the independent side-effect check. The first run that reported completion is always the
+baseline. A later completion counts as a duplicate unless its run answered a comment as above.
+
The completion time comes from the product's own record, in this order:
1. the earliest accepted run result whose `reportedWorkDisposition` is `done`;
@@ -1691,6 +1761,19 @@ its workload. They catch a stuck or misleading task that no scenario anticipated
| U4 stale notice | a Retry or failed notice stays on a task after its latest run succeeded, longer than the bound or at rest | 60 s (`RESTART_CHAOS_U4_STALE_SECONDS`) |
| U5 run budget | one task created more runs than the bound | 10 (`RESTART_CHAOS_U5_MAX_RUNS`) |
+When the workload declares a queue (staging-mix `--mix-queue` above 0), a run that waits queued while its agent's
+`maxConcurrentRuns` slots are all taken is a healthy queued run. U1 then allows it the declared queue bound,
+ceil(queue / runs) × (longest turn + 5 min); for L1 (20 runs, 20 queued, 12–22 minute turns) that is 27 minutes.
+Only queued time with a free slot of its agent counts toward the 5-minute queue bound, so a run left queued while
+its agent has a free slot still fails. Slot use comes from `wake-evidence/agent-slots.json` (every run's start and
+finish and each agent's effective cap as the server computes it, `maxConcurrentRunsSql`, so a configured 100 is saved
+as the server's limit of 50; captured at wave end). Without that file (older waves) only the declared bound applies.
+
+A parent whose turn ended `yielded` or `blocked` and that waits, with no run, until its subtasks complete is in a
+named wait (`waitingOnChildren`): R5 and U1 do not count it as a stall. The observer keeps watching such a parent
+while any subtask is open. When an older record rested during that wait and the parent then completed through its
+`issue_children_completed` wake, grading takes the completion from the wake evidence.
+
U1, U2, U4 and U5 fail the task (signature `U:...`). U3 is a warning: it is reported in each grade
(`grade.universal`), the ledger (`universal`, `universalWarnings`) and the wave's `SUMMARY.md`, and
never fails a task. U3 reads run error text and recovery-action causes from the wave database at
diff --git a/tests/restart-chaos/agent-slots.ts b/tests/restart-chaos/agent-slots.ts
new file mode 100644
index 00000000000..a13ed7a9c9e
--- /dev/null
+++ b/tests/restart-chaos/agent-slots.ts
@@ -0,0 +1,35 @@
+import type { SlotFree } from "./universal.js";
+
+export const AGENT_SLOTS_FILE = "agent-slots.json";
+
+export interface AgentSlots {
+ agents: Array<{ agentId: string; maxConcurrentRuns: number | null }>;
+ runs: Array<{ runId: string; agentId: string; startedAt: number | null; finishedAt: number | null }>;
+}
+
+export function slotFreeFrom(slots: AgentSlots | null): SlotFree {
+ if (!slots) return () => null;
+ const caps = new Map(slots.agents.map(agent => [agent.agentId, agent.maxConcurrentRuns]));
+ const agentOf = new Map(slots.runs.map(run => [run.runId, run.agentId]));
+ const byAgent = new Map();
+ for (const run of slots.runs) byAgent.set(run.agentId, [...byAgent.get(run.agentId) ?? [], run]);
+ return (runId, at) => {
+ const agentId = agentOf.get(runId);
+ const cap = agentId ? caps.get(agentId) : undefined;
+ if (!agentId || typeof cap !== "number" || cap < 1) return null;
+ const running = (byAgent.get(agentId) ?? []).filter(run => run.runId !== runId && run.startedAt !== null && run.startedAt <= at
+ && (run.finishedAt === null || run.finishedAt > at)).length;
+ return running < cap;
+ };
+}
+
+export function agentSlotsFromRows(agents: ReadonlyArray>, runs: ReadonlyArray>): AgentSlots {
+ const ms = (value: unknown) => value === null || value === undefined ? null : Number(value);
+ return {
+ agents: agents.map(agent => {
+ const cap = Number(agent.maxConcurrentRuns);
+ return { agentId: String(agent.id), maxConcurrentRuns: Number.isInteger(cap) && cap > 0 ? cap : null };
+ }),
+ runs: runs.filter(run => run.agentId).map(run => ({ runId: String(run.id), agentId: String(run.agentId), startedAt: ms(run.startedAt), finishedAt: ms(run.finishedAt) })),
+ };
+}
diff --git a/tests/restart-chaos/child-wait.test.ts b/tests/restart-chaos/child-wait.test.ts
new file mode 100644
index 00000000000..a81b1db9281
--- /dev/null
+++ b/tests/restart-chaos/child-wait.test.ts
@@ -0,0 +1,58 @@
+import { describe, expect, it } from "vitest";
+import { withChildWaits } from "./child-wait.js";
+import { gradeTask } from "./grader.js";
+import type { WakeEvidenceFile } from "./staging-mix-retry.js";
+import type { RunSample, TaskRecord, TaskSample } from "./types.js";
+
+const T0 = Date.parse("2026-11-13T05:37:21Z");
+const SEC = 1_000;
+const iso = (offset: number) => new Date(T0 + offset).toISOString();
+
+const run = (id: string, status: string, started: number, finished: number | null): RunSample => ({ id, status, createdAt: iso(started), startedAt: iso(started),
+ finishedAt: finished === null ? null : iso(finished), errorCode: null, retryOfRunId: null, wakeCommentIds: [], livenessState: null,
+ lastEvidenceAt: status === "running" ? iso(started) : null });
+const sample = (at: number, runs: RunSample[], issueStatus = "in_progress"): TaskSample => ({ at: T0 + at, reachable: true, serverReady: true, issueStatus,
+ interruption: null, activeRecoveryAction: false, executionBlocker: false, scheduledRetry: false, runs });
+
+function parentRestedWaitingOnSubtasks(): TaskRecord {
+ const samples: TaskSample[] = [];
+ for (let at = 0; at < 2_000 * SEC; at += 10 * SEC) samples.push(sample(at, [{ ...run("r1", "running", 0, null), lastEvidenceAt: iso(at) }]));
+ for (let at = 2_010 * SEC; at <= 2_210 * SEC; at += 10 * SEC) samples.push(sample(at, [run("r1", "succeeded", 0, 2_000 * SEC)]));
+ return { plan: { taskId: "t001", index: 0, workload: "SM", adapter: "opencode", fault: "K13", fireAfterMs: 0, holdMs: 0, followUpAfterMs: null, nonce: "90a3899c" },
+ issueId: "issue-1", identifier: "CHA-1", createdAt: T0, launchError: null, samples, retry: null, restAt: T0 + 2_010 * SEC, finalStatus: "in_progress",
+ deadlineAt: T0 + 7_200 * SEC, timedOut: false, followUp: null, faultIds: [], sweep: { activeRuns: [], runningWorkspaceOperations: [], activeRecoveryAction: false, error: null },
+ preFaultFiles: null, lostFiles: null };
+}
+
+const evidence = (withChildCompletion: boolean): WakeEvidenceFile => ({
+ completedAt: T0 + 2_871 * SEC,
+ runs: [{ runId: "r1", createdAt: T0, startedAt: T0, finishedAt: T0 + 2_000 * SEC, invocationSource: "assignment", triggerDetail: "system", wakeupRequestId: "w1",
+ acceptedDisposition: "yielded", acceptedAt: T0 + 1_999 * SEC },
+ ...(withChildCompletion ? [{ runId: "r3", createdAt: T0 + 2_667 * SEC, startedAt: T0 + 2_820 * SEC, finishedAt: T0 + 2_871 * SEC, invocationSource: "automation",
+ triggerDetail: "system", wakeupRequestId: "w3", acceptedDisposition: "done", acceptedAt: T0 + 2_870 * SEC }] : [])],
+ wakes: [{ id: "w1", reason: "issue_assigned", source: "assignment", triggerDetail: "system", requestedByActorType: "user", requestedByActorId: "local-board", createdAt: T0 },
+ ...(withChildCompletion ? [{ id: "w3", reason: "issue_children_completed", source: "automation", triggerDetail: "system", requestedByActorType: "agent",
+ requestedByActorId: "a1", createdAt: T0 + 2_648 * SEC }] : [])],
+});
+
+const pass = { verdict: "pass" as const, checks: [] };
+
+describe("a parent waiting on its subtasks (w-20261113T052703-s7301 t001)", () => {
+ it("is a named wait, and the parent completed by its children-completed wake rests done", () => {
+ const before = gradeTask(parentRestedWaitingOnSubtasks(), [], 4_345 * SEC, pass);
+ expect([before.rules.R1.detail, before.rules.R3.verdict, before.rules.R5.verdict, before.universal!.U1.verdict])
+ .toEqual(["rested_not_done (status in_progress, no Retry offered)", "fail", "fail", "fail"]);
+ const record = withChildWaits(parentRestedWaitingOnSubtasks(), evidence(true));
+ expect(record).toMatchObject({ finalStatus: "done", restAt: T0 + 2_871 * SEC });
+ expect(record.samples.filter(entry => entry.waitingOnChildren).length).toBe(21);
+ const after = gradeTask(record, [], 4_345 * SEC, pass);
+ expect([after.rules.R1.verdict, after.rules.R3.verdict, after.rules.R5.verdict, after.universal!.U1.verdict, after.pass]).toEqual(["pass", "n/a", "pass", "pass", true]);
+ });
+
+ it("changes nothing when no children-completed wake followed the parent's yielded turn", () => {
+ const record = parentRestedWaitingOnSubtasks();
+ expect(withChildWaits(record, evidence(false))).toBe(record);
+ expect(withChildWaits(record, null)).toBe(record);
+ expect(gradeTask(withChildWaits(record, evidence(false)), [], 4_345 * SEC, pass).pass).toBe(false);
+ });
+});
diff --git a/tests/restart-chaos/child-wait.ts b/tests/restart-chaos/child-wait.ts
new file mode 100644
index 00000000000..1b3383fdebd
--- /dev/null
+++ b/tests/restart-chaos/child-wait.ts
@@ -0,0 +1,38 @@
+import type { TaskRecord, TaskSample } from "./types.js";
+import type { WakeEvidenceFile } from "./staging-mix-retry.js";
+
+const ACTIVE = new Set(["queued", "running", "scheduled_retry"]);
+const WAITING_DISPOSITIONS = new Set(["yielded", "blocked"]);
+const CHILDREN_COMPLETED = "issue_children_completed";
+
+function childWaitWindows(evidence: WakeEvidenceFile): Array<{ from: number; until: number }> {
+ const completions = (evidence.wakes ?? []).filter(wake => wake.reason === CHILDREN_COMPLETED && typeof wake.createdAt === "number")
+ .map(wake => wake.createdAt!).sort((a, b) => a - b);
+ return (evidence.runs ?? []).flatMap(run => {
+ if (!WAITING_DISPOSITIONS.has(run.acceptedDisposition ?? "") || typeof run.finishedAt !== "number") return [];
+ const until = completions.find(at => at >= run.finishedAt!);
+ return until === undefined ? [] : [{ from: run.finishedAt, until }];
+ });
+}
+
+function idle(sample: TaskSample): boolean {
+ return sample.reachable && !["done", "cancelled"].includes(sample.issueStatus ?? "") && !(sample.runs ?? []).some(run => ACTIVE.has(run.status));
+}
+
+export function withChildWaits(record: TaskRecord, evidence: WakeEvidenceFile | null): TaskRecord {
+ if (!evidence) return record;
+ const windows = childWaitWindows(evidence);
+ if (!windows.length) return record;
+ const waiting = (sample: TaskSample) => idle(sample) && windows.some(window => sample.at >= window.from && sample.at <= window.until);
+ const samples = record.samples.map(sample => waiting(sample) ? { ...sample, waitingOnChildren: true } : sample);
+ const last = [...samples].reverse().find(sample => sample.reachable);
+ const wakes = new Map((evidence.wakes ?? []).map(wake => [wake.id, wake]));
+ const finisher = (evidence.runs ?? []).find(run => run.acceptedDisposition === "done" && run.wakeupRequestId
+ && wakes.get(run.wakeupRequestId)?.reason === CHILDREN_COMPLETED);
+ const completedAt = evidence.completedAt ?? null;
+ const restedWaiting = record.finalStatus !== "done" && record.restAt !== null && last?.waitingOnChildren === true;
+ if (restedWaiting && finisher && completedAt !== null && completedAt > record.restAt!) {
+ return { ...record, samples, finalStatus: "done", restAt: completedAt };
+ }
+ return { ...record, samples };
+}
diff --git a/tests/restart-chaos/daytona-cleanup.test.ts b/tests/restart-chaos/daytona-cleanup.test.ts
new file mode 100644
index 00000000000..773f7e96cab
--- /dev/null
+++ b/tests/restart-chaos/daytona-cleanup.test.ts
@@ -0,0 +1,48 @@
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+import { finishDaytonaCleanup } from "./daytona-cleanup.js";
+import type { DaytonaSweepResult, WaveCleanup } from "./daytona-sweep.js";
+
+const sweep: DaytonaSweepResult = { companyId: "c", environmentId: "e", checkedAt: "now", liveSandboxes: ["s1"], orphanSandboxes: [], heldLeases: [],
+ retainedLeases: [], archivedSandboxes: ["s1"], error: null };
+const clean: WaveCleanup = { environmentDeleted: true, environmentDeleteBlocked: null, remainingSandboxes: [], archivedSandboxes: ["s1:archived"],
+ deletedSandboxes: ["s1:archived"], undeletedSandboxes: [], errors: [] };
+
+async function run(cleanup: WaveCleanup, saved: DaytonaSweepResult | null) {
+ const waveDir = await mkdtemp(path.join(os.tmpdir(), "daytona-cleanup-"));
+ const kinds: string[] = [], lines: string[] = [];
+ let swept = 0;
+ try {
+ const outcome = await finishDaytonaCleanup({ sweeper: { sweep: async () => { swept += 1; return sweep; }, cleanUp: async () => cleanup }, sweep: saved, waveDir,
+ timeline: { add: kind => { kinds.push(kind); } }, log: line => { lines.push(line); } });
+ const file = JSON.parse(await readFile(path.join(waveDir, "sandbox-sweep.json"), "utf8"));
+ return { outcome, kinds, lines, swept, file };
+ } finally {
+ await rm(waveDir, { recursive: true, force: true });
+ }
+}
+
+describe("wave-end Daytona cleanup", () => {
+ it("passes a wave whose sandboxes were all deleted, and sweeps first when the wave was interrupted", async () => {
+ const { outcome, kinds, lines, swept, file } = await run(clean, null);
+ expect(outcome).toMatchObject({ failed: false, remotePending: false, error: null, sweep });
+ expect(swept).toBe(1);
+ expect(kinds).toEqual(["daytona_cleanup"]);
+ expect(lines[0]).toContain("deleted by the harness (this wave's labels only): s1:archived; left after deletion: none");
+ expect(file).toEqual({ sweep, cleanup: clean });
+ });
+
+ it("fails the wave for a sandbox left after deletion, a blocked environment delete or a still-running sandbox, and reports errors", async () => {
+ expect((await run({ ...clean, undeletedSandboxes: ["s1:archived"] }, sweep)).outcome).toMatchObject({ failed: true, remotePending: true });
+ const blocked = await run({ ...clean, environmentDeleted: false, environmentDeleteBlocked: "409" }, sweep);
+ expect(blocked.outcome).toMatchObject({ failed: true, remotePending: true });
+ expect(blocked.kinds).toEqual(["daytona_cleanup", "daytona_environment_delete_blocked"]);
+ expect(blocked.swept).toBe(0);
+ expect((await run({ ...clean, remainingSandboxes: ["s2:started"] }, sweep)).outcome.failed).toBe(true);
+ const errored = await run({ ...clean, errors: ["sandbox delete s1: boom"] }, sweep);
+ expect(errored.outcome).toMatchObject({ failed: false, remotePending: true });
+ expect(errored.outcome.error?.message).toBe("Daytona cleanup incomplete: sandbox delete s1: boom");
+ });
+});
diff --git a/tests/restart-chaos/daytona-cleanup.ts b/tests/restart-chaos/daytona-cleanup.ts
new file mode 100644
index 00000000000..8dfb0ee2b20
--- /dev/null
+++ b/tests/restart-chaos/daytona-cleanup.ts
@@ -0,0 +1,35 @@
+import { writeFile } from "node:fs/promises";
+import path from "node:path";
+import type { DaytonaSweepResult, DaytonaWaveSweeper, WaveCleanup } from "./daytona-sweep.js";
+
+export interface DaytonaCleanupOutcome {
+ sweep: DaytonaSweepResult;
+ cleanup: WaveCleanup;
+ failed: boolean;
+ remotePending: boolean;
+ error: Error | null;
+}
+
+const list = (values: readonly string[]) => values.join(", ") || "none";
+
+export function daytonaCleanupLine(sweep: DaytonaSweepResult, cleanup: WaveCleanup): string {
+ return `Daytona R6 sweep: ${sweep.liveSandboxes.length} live, ${sweep.archivedSandboxes.length} archived after release, `
+ + `${sweep.orphanSandboxes.length} orphaned, ${sweep.heldLeases.length} held leases${sweep.error ? `, error ${sweep.error}` : ""}; `
+ + `environment deleted: ${cleanup.environmentDeleted}${cleanup.environmentDeleteBlocked ? " (still blocked by a pending sandbox cleanup)" : ""}; `
+ + `still running after cleanup: ${list(cleanup.remainingSandboxes)}; archived: ${list(cleanup.archivedSandboxes)}; `
+ + `deleted by the harness (this wave's labels only): ${list(cleanup.deletedSandboxes)}; left after deletion: ${list(cleanup.undeletedSandboxes)}`;
+}
+
+export async function finishDaytonaCleanup(input: { sweeper: Pick; sweep: DaytonaSweepResult | null; waveDir: string;
+ timeline: { add(kind: string, data: Record): void }; log?: (line: string) => void }): Promise {
+ const sweep = input.sweep ?? await input.sweeper.sweep(0);
+ const cleanup = await input.sweeper.cleanUp();
+ input.timeline.add("daytona_cleanup", cleanup);
+ (input.log ?? console.info)(daytonaCleanupLine(sweep, cleanup));
+ if (cleanup.environmentDeleteBlocked) input.timeline.add("daytona_environment_delete_blocked", { detail: cleanup.environmentDeleteBlocked });
+ const failed = Boolean(cleanup.remainingSandboxes.length || cleanup.undeletedSandboxes.length || cleanup.environmentDeleteBlocked);
+ await writeFile(path.join(input.waveDir, "sandbox-sweep.json"), `${JSON.stringify({ sweep, cleanup }, null, 2)}\n`, { mode: 0o600 })
+ .catch(error => input.timeline.add("sandbox_sweep_write_failed", { error: String(error) }));
+ return { sweep, cleanup, failed, remotePending: failed || cleanup.errors.length > 0,
+ error: cleanup.errors.length ? new Error(`Daytona cleanup incomplete: ${cleanup.errors.join("; ")}`) : null };
+}
diff --git a/tests/restart-chaos/daytona-sweep.test.ts b/tests/restart-chaos/daytona-sweep.test.ts
index b2e6f91a9cc..e28282f7103 100644
--- a/tests/restart-chaos/daytona-sweep.test.ts
+++ b/tests/restart-chaos/daytona-sweep.test.ts
@@ -1,9 +1,10 @@
import { describe, expect, it } from "vitest";
import {
- assertHarnessMayMutate, DaytonaWaveSweeper, isProtectedSandbox, leaksForTask, readOnlyDaytona, reconcileSandboxes, selectWaveSandboxes,
- PENDING_CLEANUP_RETRY_MS, R6_SETTLE_MS, sweepHasLeaks, unattributedLeaks, type DaytonaSandboxRow, type DaytonaSweepResult,
+ assertHarnessMayMutate, daytonaRestDelete, DaytonaWaveSweeper, isProtectedSandbox, leaksForTask, readOnlyDaytona, reconcileSandboxes, selectWaveSandboxes,
+ PENDING_CLEANUP_RETRY_MS, R6_SETTLE_MS, SANDBOX_DELETE_MIN_TIMEOUT_MS, sweepHasLeaks, unattributedLeaks, type DaytonaSandboxRow, type DaytonaSweepResult,
} from "./daytona-sweep.js";
import { ENVIRONMENT_CLEANUP_RETRY_MS } from "../../server/src/services/environment-cleanup-state.js";
+import { gradeWarmShutdown } from "./warm-shutdown.js";
describe("R6 settle window", () => {
it("covers the product's pending-cleanup retry backoff plus a sweep, capped at 8 minutes", () => {
@@ -16,6 +17,7 @@ describe("R6 settle window", () => {
describe("cleanup through Paperclip", () => {
const COMPANY = "11111111-1111-4111-8111-111111111111";
const ENVIRONMENT = "22222222-2222-4222-8222-222222222222";
+ const OTHER_COMPANY = "33333333-3333-4333-8333-333333333333";
const listing = { list: () => (async function* () {})() };
it("treats a 409 pending-cleanup environment delete as blocked, not a harness error, and retries it", async () => {
@@ -29,6 +31,76 @@ describe("cleanup through Paperclip", () => {
expect(calls).toBeGreaterThan(1);
});
+ const waveRows = (states: string[], extra: Partial = {}): DaytonaSandboxRow[] => states.map(state =>
+ ({ id: state, name: `paperclip-create-${state}`, state, labels: { "paperclip-company-id": COMPANY, "paperclip-environment-id": ENVIRONMENT }, ...extra }));
+ const deletingDaytona = (rows: DaytonaSandboxRow[]) => {
+ const removed: string[] = [];
+ const client = readOnlyDaytona({ list: () => (async function* () { yield* rows.map(row => ({ ...row })); })() });
+ const removeSandbox = async (id: string) => {
+ removed.push(id);
+ const row = rows.find(candidate => candidate.id === id);
+ if (row) row.state = "destroyed";
+ };
+ return { client, removeSandbox, removed };
+ };
+
+ it("deletes the wave's archived setup sandboxes and every other sandbox carrying both wave labels, and leaves nothing behind", async () => {
+ const rows = [...waveRows(["started", "archiving", "archived", "destroyed"]),
+ { id: "fleet", name: "paperclip-create-fleet", state: "archived", labels: { "paperclip-company-id": OTHER_COMPANY, "paperclip-environment-id": ENVIRONMENT } },
+ { id: "other-env", name: "paperclip-create-x", state: "archived", labels: { "paperclip-company-id": COMPANY, "paperclip-environment-id": OTHER_COMPANY } }];
+ const { client, removeSandbox, removed } = deletingDaytona(rows);
+ const sweeper = new DaytonaWaveSweeper({ apiKey: "unused", companyId: COMPANY, environmentId: ENVIRONMENT, api: { delete: async () => undefined } as never, removeSandbox });
+ (sweeper as unknown as { client: unknown }).client = client;
+ const result = await sweeper.cleanUp(0, 1_000);
+ expect(result.remainingSandboxes).toEqual(["started:started"]);
+ expect(result.archivedSandboxes).toEqual(["archiving:archiving", "archived:archived"]);
+ expect(removed.sort()).toEqual(["archived", "archiving", "started"]);
+ expect(result.deletedSandboxes.sort()).toEqual(["archived:archived", "archiving:archiving", "started:started"]);
+ expect(result.undeletedSandboxes).toEqual([]);
+ expect(rows.find(row => row.id === "fleet")!.state).toBe("archived");
+ expect(rows.find(row => row.id === "other-env")!.state).toBe("archived");
+ });
+
+ it("fails cleanup when a wave sandbox survives deletion, and never deletes a fleet snapshot even with the wave's labels", async () => {
+ const rows = [...waveRows(["archived"]), ...waveRows(["fleet"], { snapshot: "fleet-sbx-9", state: "archived" })];
+ const { client, removeSandbox, removed } = deletingDaytona(rows);
+ const sweeper = new DaytonaWaveSweeper({ apiKey: "unused", companyId: COMPANY, environmentId: ENVIRONMENT, api: { delete: async () => undefined } as never, removeSandbox });
+ (sweeper as unknown as { client: unknown }).client = client;
+ const result = await sweeper.cleanUp(0, 1_000);
+ expect(removed).toEqual(["archived"]);
+ expect(result.undeletedSandboxes).toEqual(["fleet:archived"]);
+ expect(result.errors.join(" ")).toMatch(/sandbox delete fleet: Refusing to change protected/);
+ expect(gradeWarmShutdown({ cleanup: result, serverLog: "" })).toMatchObject({ verdict: "fail", leakedSandboxes: ["fleet:archived"] });
+ });
+
+ it("sends the Daytona delete for one sandbox id with the wave's key", async () => {
+ const calls: Array<{ url: string; init: RequestInit | undefined }> = [];
+ const remove = daytonaRestDelete("key-1", "https://daytona.test/api", (async (url: string, init?: RequestInit) => {
+ calls.push({ url, init });
+ return new Response(null, { status: url.endsWith("gone") ? 404 : url.endsWith("bad") ? 500 : 200 });
+ }) as typeof fetch);
+ const signal = new AbortController().signal;
+ await remove("abc", signal);
+ await remove("gone", signal);
+ await expect(remove("bad", signal)).rejects.toThrow(/returned 500/);
+ expect(calls[0]).toEqual({ url: "https://daytona.test/api/sandbox/abc", init: { method: "DELETE", headers: { Authorization: "Bearer key-1" }, signal } });
+ });
+
+ it("aborts a hung delete at the cleanup deadline and reports the sandbox as left", async () => {
+ const rows = waveRows(["archived"]);
+ const client = readOnlyDaytona({ list: () => (async function* () { yield* rows.map(row => ({ ...row })); })() });
+ const removeSandbox = (_id: string, signal: AbortSignal) => new Promise((_resolve, reject) => {
+ signal.addEventListener("abort", () => reject(new Error("delete aborted")), { once: true });
+ });
+ const sweeper = new DaytonaWaveSweeper({ apiKey: "unused", companyId: COMPANY, environmentId: ENVIRONMENT, api: { delete: async () => undefined } as never, removeSandbox });
+ (sweeper as unknown as { client: unknown }).client = client;
+ const started = Date.now();
+ const result = await sweeper.cleanUp(0, 1_000);
+ expect(Date.now() - started).toBeLessThan(SANDBOX_DELETE_MIN_TIMEOUT_MS + 3_000);
+ expect(result.undeletedSandboxes).toEqual(["archived:archived"]);
+ expect(result.errors.join(" ")).toMatch(/sandbox delete archived: delete aborted/);
+ });
+
it("reports other environment delete failures as errors", async () => {
const api = { delete: async () => { throw new Error("DELETE returned 500: boom"); } };
const sweeper = new DaytonaWaveSweeper({ apiKey: "unused", companyId: COMPANY, environmentId: ENVIRONMENT, api: api as never });
@@ -79,13 +151,13 @@ describe("wave sandbox selection is read-only and exact", () => {
expect(() => readOnlyDaytona({} as never)).toThrow(/no list\(\)/);
});
- it("hard-refuses fleet sandboxes and anything not labelled for the wave, even with matching labels", () => {
+ it("hard-refuses fleet snapshots and anything not labelled for the wave; a plugin-created name with the wave's labels is the wave's own", () => {
const waveLabels = { "paperclip-company-id": COMPANY, "paperclip-environment-id": ENVIRONMENT };
- expect(isProtectedSandbox({ name: "paperclip-create-1" })).toBe(true);
expect(isProtectedSandbox({ name: "x", snapshot: "fleet-sbx-2" })).toBe(true);
- expect(() => assertHarnessMayMutate(labelled("a", waveLabels, { name: "paperclip-create-1" }), wave)).toThrow(/protected/);
expect(() => assertHarnessMayMutate(labelled("b", waveLabels, { name: "probe", snapshot: "fleet-sbx-2" }), wave)).toThrow(/protected/);
expect(() => assertHarnessMayMutate(labelled("c", {}, { name: "probe" }), wave)).toThrow(/not labelled for this wave/);
+ expect(() => assertHarnessMayMutate(labelled("e", { "paperclip-company-id": OTHER }, { name: "paperclip-create-1" }), wave)).toThrow(/not labelled for this wave/);
+ expect(() => assertHarnessMayMutate(labelled("a", waveLabels, { name: "paperclip-create-1" }), wave)).not.toThrow();
expect(() => assertHarnessMayMutate(labelled("d", waveLabels, { name: "probe" }), wave)).not.toThrow();
});
});
diff --git a/tests/restart-chaos/daytona-sweep.ts b/tests/restart-chaos/daytona-sweep.ts
index d1d434ff51e..2e31241faee 100644
--- a/tests/restart-chaos/daytona-sweep.ts
+++ b/tests/restart-chaos/daytona-sweep.ts
@@ -2,12 +2,6 @@
* R6 for `--env daytona`: reconcile the Daytona sandboxes this wave created
* against the instance's environment leases and report leaks.
*
- * The Daytona org is shared with the live Cloud fleet, so this module NEVER
- * changes a sandbox. Its client exposes only `list`; stop, archive and delete
- * do not exist here. Cleanup goes only through Paperclip: deleting the wave's
- * own environment, which acts on the leases in the wave's isolated database.
- * Leaks are reported for a person, and expire through the environment's ttl.
- *
* Wave ownership comes from the labels the Daytona plugin puts on every
* sandbox it creates. Each wave's isolated instance mints a fresh company and
* environment UUID, and a row is selected only when both labels equal them.
@@ -129,7 +123,6 @@ const CLEANUP_SWEEP_SLACK_MS = 90_000;
/** R6 waits this long for leases and sandboxes to settle before calling them leaks. */
export const R6_SETTLE_MS = Math.min(8 * 60_000, PENDING_CLEANUP_RETRY_MS + CLEANUP_SWEEP_SLACK_MS);
-/** The only Daytona operation the harness may perform. */
export interface ReadOnlyDaytona { list(query: { labels: Record }): AsyncIterable }
/**
@@ -142,22 +135,34 @@ export function readOnlyDaytona(client: { list(query?: { labels?: Record }) => client.list(query) });
}
-/** Fleet sandboxes the harness must never act on, even if their labels matched. */
export function isProtectedSandbox(sandbox: Pick): boolean {
- return (sandbox.name ?? "").startsWith("paperclip-create-") || (sandbox.snapshot ?? "").startsWith("fleet-sbx-");
+ return (sandbox.snapshot ?? "").startsWith("fleet-sbx-");
}
-/**
- * Hard guard for any future mutating path. It refuses `paperclip-create-*`
- * names and `fleet-sbx-*` snapshots outright, and anything not carrying both
- * of the wave's UUID labels. Every plugin-created sandbox is
- * `paperclip-create-*`, so the harness can never mutate one directly.
- */
export function assertHarnessMayMutate(sandbox: DaytonaSandboxRow, wave: { companyId: string; environmentId: string }): void {
if (isProtectedSandbox(sandbox)) throw new Error(`Refusing to change protected Daytona sandbox ${sandbox.id} (${sandbox.name ?? sandbox.snapshot})`);
if (selectWaveSandboxes([sandbox], wave).length !== 1) throw new Error(`Refusing to change Daytona sandbox ${sandbox.id}: it is not labelled for this wave`);
}
+export type SandboxRemove = (sandboxId: string, signal: AbortSignal) => Promise;
+export type WaveSandboxDelete = (sandbox: DaytonaSandboxRow, signal: AbortSignal) => Promise;
+
+export const SANDBOX_DELETE_MIN_TIMEOUT_MS = 5_000;
+
+export function waveSandboxDeleter(wave: { companyId: string; environmentId: string }, remove: SandboxRemove): WaveSandboxDelete {
+ return Object.freeze(async (sandbox: DaytonaSandboxRow, signal: AbortSignal) => {
+ assertHarnessMayMutate(sandbox, wave);
+ await remove(sandbox.id, signal);
+ });
+}
+
+export function daytonaRestDelete(apiKey: string, apiUrl = "https://app.daytona.io/api", request: typeof fetch = fetch): SandboxRemove {
+ return async (sandboxId, signal) => {
+ const response = await request(`${apiUrl}/sandbox/${encodeURIComponent(sandboxId)}`, { method: "DELETE", headers: { Authorization: `Bearer ${apiKey}` }, signal });
+ if (!response.ok && response.status !== 404) throw new Error(`DELETE sandbox ${sandboxId} returned ${response.status}`);
+ };
+}
+
const UUID = /^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/i;
/** Keeps only rows whose company AND environment labels equal this wave's UUIDs. */
@@ -169,7 +174,12 @@ export function selectWaveSandboxes(rows: readonly DaytonaSandboxRow[], wave: {
export class DaytonaWaveSweeper {
private client: ReadOnlyDaytona | null = null;
- constructor(private readonly input: { apiKey: string; companyId: string; environmentId: string; api: ChaosApi }) {}
+ private readonly deleteSandbox: WaveSandboxDelete;
+ constructor(private readonly input: { apiKey: string; companyId: string; environmentId: string; api: ChaosApi;
+ removeSandbox?: SandboxRemove }) {
+ this.deleteSandbox = waveSandboxDeleter({ companyId: input.companyId, environmentId: input.environmentId },
+ input.removeSandbox ?? daytonaRestDelete(input.apiKey));
+ }
private async daytona(): Promise {
this.client ??= readOnlyDaytona(await createRemoteFixtureClient(this.input.apiKey) as unknown as Parameters[0]);
@@ -205,13 +215,7 @@ export class DaytonaWaveSweeper {
}
}
- /**
- * Cleanup through Paperclip only: delete the wave's own environment (with its
- * reusable leases). Then list, read-only, what is still not destroyed or
- * archived, for the report. No sandbox is changed by the harness itself.
- */
- async cleanUp(settleMs = 90_000, deleteWindowMs = R6_SETTLE_MS): Promise<{ environmentDeleted: boolean; environmentDeleteBlocked: string | null;
- remainingSandboxes: string[]; errors: string[] }> {
+ async cleanUp(settleMs = 90_000, deleteWindowMs = R6_SETTLE_MS): Promise {
const errors: string[] = [];
let environmentDeleted = false;
let environmentDeleteBlocked: string | null = null;
@@ -226,14 +230,16 @@ export class DaytonaWaveSweeper {
else errors.push(`environment delete: ${message}`);
}
let remainingSandboxes: string[] = [];
+ let archivedSandboxes: string[] = [];
// Provider teardown is asynchronous; give it a bounded settle before reporting.
const deadline = Date.now() + settleMs;
for (;;) {
try {
- remainingSandboxes = (await this.listWaveSandboxes())
- .filter(sandbox => !GONE_SANDBOX_STATES.has(String(sandbox.state ?? "").toLowerCase())
- && !ARCHIVED_SANDBOX_STATES.has(String(sandbox.state ?? "").toLowerCase()))
+ const listed = await this.listWaveSandboxes();
+ const state = (sandbox: DaytonaSandboxRow) => String(sandbox.state ?? "").toLowerCase();
+ remainingSandboxes = listed.filter(sandbox => !GONE_SANDBOX_STATES.has(state(sandbox)) && !ARCHIVED_SANDBOX_STATES.has(state(sandbox)))
.map(sandbox => `${sandbox.id}:${sandbox.state}`);
+ archivedSandboxes = listed.filter(sandbox => ARCHIVED_SANDBOX_STATES.has(state(sandbox))).map(sandbox => `${sandbox.id}:${sandbox.state}`);
} catch (error) {
errors.push(`sandbox listing: ${error instanceof Error ? error.message : String(error)}`);
break;
@@ -241,6 +247,46 @@ export class DaytonaWaveSweeper {
if (!remainingSandboxes.length || Date.now() >= deadline) break;
await delay(10_000);
}
- return { environmentDeleted, environmentDeleteBlocked, remainingSandboxes, errors };
+ const removal = await this.deleteWaveSandboxes(settleMs, errors);
+ return { environmentDeleted, environmentDeleteBlocked, remainingSandboxes, archivedSandboxes, ...removal, errors };
+ }
+
+ private async deleteWaveSandboxes(settleMs: number, errors: string[]): Promise> {
+ const deleted = new Set();
+ const failures = new Map();
+ let undeleted: DaytonaSandboxRow[] = [];
+ const deadline = Date.now() + settleMs;
+ for (let attempt = 0; ; attempt += 1) {
+ try {
+ undeleted = (await this.listWaveSandboxes()).filter(sandbox => !GONE_SANDBOX_STATES.has(String(sandbox.state ?? "").toLowerCase()));
+ } catch (error) {
+ errors.push(`sandbox listing for deletion: ${error instanceof Error ? error.message : String(error)}`);
+ return { deletedSandboxes: [...deleted], undeletedSandboxes: ["unlisted"] };
+ }
+ if (!undeleted.length || (attempt > 0 && Date.now() >= deadline)) break;
+ const pass = AbortSignal.timeout(Math.max(SANDBOX_DELETE_MIN_TIMEOUT_MS, deadline - Date.now()));
+ for (const sandbox of undeleted) {
+ try {
+ await this.deleteSandbox(sandbox, pass);
+ deleted.add(`${sandbox.id}:${sandbox.state}`);
+ failures.delete(sandbox.id);
+ } catch (error) {
+ failures.set(sandbox.id, error instanceof Error ? error.message : String(error));
+ }
+ }
+ await delay(Math.min(10_000, Math.max(0, deadline - Date.now())));
+ }
+ for (const [id, message] of failures) if (undeleted.some(sandbox => sandbox.id === id)) errors.push(`sandbox delete ${id}: ${message}`);
+ return { deletedSandboxes: [...deleted], undeletedSandboxes: undeleted.map(sandbox => `${sandbox.id}:${sandbox.state}`) };
}
}
+
+export type WaveCleanup = {
+ environmentDeleted: boolean;
+ environmentDeleteBlocked: string | null;
+ remainingSandboxes: string[];
+ archivedSandboxes: string[];
+ deletedSandboxes: string[];
+ undeletedSandboxes: string[];
+ errors: string[];
+};
diff --git a/tests/restart-chaos/follow-up-delivery.ts b/tests/restart-chaos/follow-up-delivery.ts
new file mode 100644
index 00000000000..0e907b116ec
--- /dev/null
+++ b/tests/restart-chaos/follow-up-delivery.ts
@@ -0,0 +1,64 @@
+type Json = Record;
+
+export interface DeliveryRun {
+ runId: string | null;
+ createdAt: number | null;
+ startedAt?: number | null;
+ wakeupRequestId: string | null;
+ contextSnapshot?: unknown;
+}
+
+export interface DeliveryWake { id: string | null; createdAt?: number | null; payload?: unknown }
+
+export interface LaterAnswer {
+ runId: string;
+ commentIds: string[];
+ postedAt: number | null;
+ queuedAt: number | null;
+ startedAt: number;
+}
+
+const strings = (value: unknown): string[] => Array.isArray(value) ? value.filter((item): item is string => typeof item === "string")
+ : typeof value === "string" ? [value] : [];
+const object = (value: unknown): Json => value && typeof value === "object" && !Array.isArray(value) ? value as Json : {};
+
+export function wakeInputCommentIds(run: Pick, wake: DeliveryWake | undefined): string[] {
+ const context = object(run.contextSnapshot);
+ const payload = object(wake?.payload);
+ const wakeContext = object(payload._paperclipWakeContext);
+ return [...new Set([...strings(context.wakeCommentIds), ...strings(context.wakeCommentId), ...strings(object(context.paperclipWake).commentIds),
+ ...strings(payload.commentId), ...strings(wakeContext.wakeCommentIds), ...strings(wakeContext.wakeCommentId)])];
+}
+
+export function laterAnswers(evidence: { runs?: readonly DeliveryRun[]; wakes?: readonly DeliveryWake[] } | null): Map {
+ const answers = new Map();
+ if (!evidence) return answers;
+ const wakes = new Map((evidence.wakes ?? []).map(wake => [wake.id, wake]));
+ const started = (evidence.runs ?? []).filter((run): run is DeliveryRun & { runId: string; startedAt: number } => Boolean(run.runId) && typeof run.startedAt === "number")
+ .sort((a, b) => a.startedAt - b.startedAt);
+ const firstStart = started[0]?.startedAt ?? null;
+ started.forEach((run, index) => {
+ const wake = run.wakeupRequestId ? wakes.get(run.wakeupRequestId) : undefined;
+ const owed = wakeInputCommentIds(run, wake);
+ const postedAt = typeof wake?.createdAt === "number" ? wake.createdAt : null;
+ const postedDuringWork = postedAt !== null && firstStart !== null ? postedAt >= firstStart : index > 0;
+ if (index > 0 && postedDuringWork && owed.length) {
+ answers.set(run.runId, { runId: run.runId, commentIds: owed, postedAt, queuedAt: run.createdAt, startedAt: run.startedAt });
+ }
+ });
+ return answers;
+}
+
+function percentile(values: readonly number[], fraction: number): number | null {
+ if (!values.length) return null;
+ const sorted = [...values].sort((a, b) => a - b);
+ return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(fraction * sorted.length) - 1))]!;
+}
+
+export function laterAnswerWaits(answers: readonly LaterAnswer[]): { count: number; queueWaitP50Ms: number | null; queueWaitMaxMs: number | null;
+ answerAfterPostP50Ms: number | null; answerAfterPostMaxMs: number | null } {
+ const queued = answers.flatMap(answer => answer.queuedAt === null ? [] : [answer.startedAt - answer.queuedAt]);
+ const posted = answers.flatMap(answer => answer.postedAt === null ? [] : [answer.startedAt - answer.postedAt]);
+ return { count: answers.length, queueWaitP50Ms: percentile(queued, 0.5), queueWaitMaxMs: queued.length ? Math.max(...queued) : null,
+ answerAfterPostP50Ms: percentile(posted, 0.5), answerAfterPostMaxMs: posted.length ? Math.max(...posted) : null };
+}
diff --git a/tests/restart-chaos/grader.ts b/tests/restart-chaos/grader.ts
index 8796bad647d..8c494c7070c 100644
--- a/tests/restart-chaos/grader.ts
+++ b/tests/restart-chaos/grader.ts
@@ -1,7 +1,7 @@
import { UI_RETRY_BLOCKED_BY_MODAL, UI_RETRY_BUTTON_MISSING, UI_RETRY_NO_REQUEST, type FaultReceipt, type OracleResult, type RuleId, type RuleResult, type TaskGrade, type TaskRecord, type TaskSample } from "./types.js";
import { g1Decision } from "./workloads.js";
import { gradeDiskFullRetry, isDiskFullFault } from "./disk-full-faults.js";
-import { enforcedUniversalFailures, gradeUniversal, type UniversalDbEvidence } from "./universal.js";
+import { enforcedUniversalFailures, gradeUniversal, type SlotFree, type UniversalBounds, type UniversalDbEvidence } from "./universal.js";
/** Pure R1–R6 grading over observer samples (plan §2). No I/O. */
@@ -192,7 +192,7 @@ export function gradeR4(record: TaskRecord): RuleResult {
function namedWait(sample: TaskSample): boolean {
const interruption = sample.interruption;
- return Boolean(sample.executionBlocker || sample.scheduledRetry || sample.activeRecoveryAction || interruption?.retryWait
+ return Boolean(sample.executionBlocker || sample.scheduledRetry || sample.activeRecoveryAction || sample.waitingOnChildren || interruption?.retryWait
|| interruption?.state === "retry_scheduled" || interruption?.canRetry
|| (interruption && ["failed", "waiting_for_user"].includes(interruption.state)));
}
@@ -250,7 +250,7 @@ export function gradeR6(record: TaskRecord): RuleResult {
}
export function gradeTask(record: TaskRecord, faults: readonly FaultReceipt[], budgetMs: number, oracle: OracleResult,
- universalDb?: UniversalDbEvidence): TaskGrade {
+ universalDb?: UniversalDbEvidence, queue: { bounds?: UniversalBounds; slotFree?: SlotFree } = {}): TaskGrade {
const crossing = crossingFaults(record, faults);
const rules: Record = {
R1: gradeR1(record, budgetMs, faults),
@@ -262,7 +262,7 @@ export function gradeTask(record: TaskRecord, faults: readonly FaultReceipt[], b
};
const failed = (Object.entries(rules) as Array<[RuleId, RuleResult]>).find(([, rule]) => rule.verdict === "fail");
// Universal checks run on every task whatever its workload; only enforced ones fail it (universal.ts).
- const universal = gradeUniversal(record, universalDb ? { db: universalDb } : {});
+ const universal = gradeUniversal(record, { ...(universalDb ? { db: universalDb } : {}), ...queue });
const [universalFailure] = enforcedUniversalFailures(universal);
const pass = !failed && oracle.verdict === "pass" && !universalFailure;
let signature: string | null = null;
diff --git a/tests/restart-chaos/journey-matrix.ts b/tests/restart-chaos/journey-matrix.ts
index 3d434544ebb..f063c2497d8 100644
--- a/tests/restart-chaos/journey-matrix.ts
+++ b/tests/restart-chaos/journey-matrix.ts
@@ -87,7 +87,6 @@ export function applyJourneyOptions(options: WaveOptions, input: { matrix: boole
options.tasks = matrixCells(options).length;
}
if (input.serverFrom !== undefined) {
- if (options.env !== "local") throw new Error("--server-from runs only with --env local");
if (options.faults.includes("K14")) throw new Error("--server-from runs every server from one checkout; use --upgrade-from for K14");
options.serverFrom = path.resolve(input.serverFrom);
}
diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts
index 1c319e547b2..67b9c8e0325 100644
--- a/tests/restart-chaos/main.ts
+++ b/tests/restart-chaos/main.ts
@@ -27,9 +27,11 @@ import { DiskFullFaultSeam, isDiskFullFault, receiptsForIssue, withDiskFullCheck
import { CREDENTIAL_FOR_ADAPTER, parseCommand, replayCommand, waveIdFor } from "./config.js";
import { assertControllerMatchesImage, assertProviderPackMismatch, defaultDaytonaImageCacheRoot, prepareDaytonaImageArtifacts } from "./daytona-image.js";
import { PLUGIN_BUILD_STAMP, pluginBuildState } from "./plugin-build.js";
+import { finishDaytonaCleanup } from "./daytona-cleanup.js";
import { DaytonaWaveSweeper, leaksForTask, unattributedLeaks, type DaytonaSweepResult } from "./daytona-sweep.js";
import { knownSecretValues, redactKnownSecrets, wakeEvidenceOnce, writeEvidenceBundle } from "./evidence.js";
import { gradeTask } from "./grader.js";
+import { loadWaveGradeInputs } from "./wave-grade-inputs.js";
import { releaseInstance } from "../runner-e2e/instance-retention.js";
import { checkPlanInvariants, readUniversalDbEvidence, summarizeViolations, type InvariantCheck } from "./invariants.js";
import { collectContentionEvidence, RunRowPressure } from "./contention.js";
@@ -140,7 +142,7 @@ async function prepareDaytona(plan: WavePlan, timeline: Timeline): Promise console.info(`[daytona-image] ${line}`) });
- const { localContentId } = await assertControllerMatchesImage(artifacts, repositoryRoot);
+ const { localContentId } = await assertControllerMatchesImage(artifacts, plan.options.serverFrom ?? repositoryRoot);
timeline.add("daytona_image_matched", { image, contentId: artifacts.contentId, localContentId, sourceRevision: artifacts.sourceRevision,
runnerdSha256: artifacts.runnerdSha256, providerPackDigest: artifacts.providerPackDigest, platformManifestDigest: artifacts.platformManifestDigest });
ensureDaytonaPlugin(timeline);
@@ -789,6 +791,7 @@ async function main(): Promise {
const universalByIssue = await readUniversalDbEvidence(evidenceDb, lab.companyId).catch(error => {
timeline.add("universal_db_evidence_failed", { error: String(error) }); return null; });
const allRunMessages = new Map([...(universalByIssue?.values() ?? [])].flatMap(entry => [...entry.runMessages]));
+ const gradeInputs = await loadWaveGradeInputs(waveDir, plan.options.stagingMix);
const costSummary = launched.some(observer => observer.plan.workload === "L1")
? await api.get>(`/api/companies/${lab.companyId}/costs/summary?period=all`)
.then(summary => typeof summary.spendCents === "number"
@@ -827,8 +830,9 @@ async function main(): Promise {
observer.record.faultIds, parseRecoveryStepReceipts(await readFile(recoveryStepReceipts, "utf8").catch(() => "")), artifacts?.runs ?? []);
// CR1–CR3: the task's own fault applied (CR2 killed a provider, CR3 stalled the plugin worker; class-r.ts).
const toolOracle = withClassRCheck(stepOracle, observer.plan, faults);
+ Object.assign(observer.record, await gradeInputs.prepare(observer.plan.taskId, observer.record));
const grade = gradeTask(observer.record, faults, observer.definition.budgetMs, toolOracle, universalByIssue ? {
- runMessages: allRunMessages, recoveryActions: universalByIssue.get(observer.record.issueId ?? "")?.recoveryActions ?? [] } : undefined);
+ runMessages: allRunMessages, recoveryActions: universalByIssue.get(observer.record.issueId ?? "")?.recoveryActions ?? [] } : undefined, gradeInputs.queue);
let evidence: string | null = null;
if (!grade.pass) {
anyFailure = true;
@@ -916,31 +920,12 @@ async function main(): Promise {
await board?.close().catch(() => undefined);
await closeJourneyApps();
try { if (sweeper) try {
- const owned = sweeper as DaytonaWaveSweeper;
- // An interrupted wave still reports what it leaked before cleanup.
- sandboxSweep ??= await owned.sweep(0);
- // Cleanup only through Paperclip; the harness never changes a sandbox.
- const cleanup = await owned.cleanUp();
- daytonaCleanup = cleanup;
- timeline.add("daytona_cleanup", cleanup);
- console.info(`Daytona R6 sweep: ${sandboxSweep.liveSandboxes.length} live, ${sandboxSweep.archivedSandboxes.length} archived after release, `
- + `${sandboxSweep.orphanSandboxes.length} orphaned, `
- + `${sandboxSweep.heldLeases.length} held leases${sandboxSweep.error ? `, error ${sandboxSweep.error}` : ""}; `
- + `environment deleted: ${cleanup.environmentDeleted}${cleanup.environmentDeleteBlocked ? " (still blocked by a pending sandbox cleanup)" : ""}; still running after cleanup: ${cleanup.remainingSandboxes.join(", ") || "none"}`);
- // A sandbox neither archived nor destroyed after cleanup is a leak for a person to act on.
- if (cleanup.remainingSandboxes.length) { anyFailure = true; remoteCleanupPending = true; }
- // A delete still blocked by pending sandbox cleanup after the settle window
- // is a leak, not a harness error: fail the wave and keep the instance, whose
- // database owns that cleanup work.
- if (cleanup.environmentDeleteBlocked) {
- anyFailure = true;
- remoteCleanupPending = true;
- timeline.add("daytona_environment_delete_blocked", { detail: cleanup.environmentDeleteBlocked });
- }
- if (cleanup.errors.length) remoteCleanupPending = true;
- if (cleanup.errors.length) harnessError ??= new Error(`Daytona cleanup incomplete: ${cleanup.errors.join("; ")}`);
- await writeFile(path.join(waveDir, "sandbox-sweep.json"), `${JSON.stringify({ sweep: sandboxSweep, cleanup }, null, 2)}\n`, { mode: 0o600 })
- .catch(error => timeline.add("sandbox_sweep_write_failed", { error: String(error) }));
+ const outcome = await finishDaytonaCleanup({ sweeper: sweeper as DaytonaWaveSweeper, sweep: sandboxSweep, waveDir, timeline });
+ sandboxSweep = outcome.sweep;
+ daytonaCleanup = outcome.cleanup;
+ if (outcome.failed) anyFailure = true;
+ if (outcome.remotePending) remoteCleanupPending = true;
+ if (outcome.error) harnessError ??= outcome.error;
} catch (error) {
harnessError ??= error;
remoteCleanupPending = true;
diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts
index c24c7e7dd25..ec339633307 100644
--- a/tests/restart-chaos/observer.ts
+++ b/tests/restart-chaos/observer.ts
@@ -52,6 +52,11 @@ export async function listWorkspaceFiles(root: string): Promise {
export const QUEUED_TRIGGER_MAX_EMPTY_SAMPLES = 3;
+export function bannerRestDue(since: number, now: number, raw = process.env.RESTART_CHAOS_MIX_BANNER_REST_SECONDS): boolean {
+ if (raw === undefined || raw === "" || !/^[0-9]+$/.test(raw)) return false;
+ return now - since >= Number(raw) * 1000;
+}
+
function stillQueued(runs: readonly RunSample[]): boolean {
return runs.some(run => run.status === "queued" && !run.startedAt) && !runs.some(run => run.startedAt || run.status === "running");
}
@@ -207,6 +212,7 @@ export class TaskObserver {
readonly record: TaskRecord;
issue: Json | null = null;
private restCandidateSince: number | null = null;
+ private bannerSince: number | null = null;
private childrenSettledAt: number | null = null;
private childStallCheckedAt: number | null = null;
private stallSince: number | null = null;
@@ -513,6 +519,10 @@ export class TaskObserver {
this.restCandidateSince = null;
return;
}
+ if (isStalledSample(sample) && this.definition.collect.childDocuments && this.record.issueId) {
+ const children = await this.collectChildren(this.record.issueId).catch(() => null);
+ if (children?.length && childrenPending(children)) sample.waitingOnChildren = true;
+ }
// A task waiting on the board (G1's approval, H1's paused company) is in a named wait, not a stall.
this.stallSince = isStalledSample(sample) && !this.awaitingBoard() ? (this.stallSince ?? now) : null;
if (this.stallSince !== null && now - this.stallSince > STALL_GIVE_UP_MS) {
@@ -693,8 +703,13 @@ export class TaskObserver {
if (status !== "done" && (!this.record.retry || faultSinceRetry) && sample.interruption?.canRetry) {
if (this.plan.stagingMix?.retryDuringWave && holdsRetry(this.record, this.plan.stagingMix.holdMs, now)) { this.restCandidateSince = null; return; }
if (!this.plan.stagingMix || this.plan.stagingMix.retryDuringWave) { await this.boardRetry(sample, now); return; }
- this.restCandidateSince = null;
- return;
+ this.bannerSince ??= now;
+ if (!bannerRestDue(this.bannerSince, now)) {
+ this.restCandidateSince = null;
+ return;
+ }
+ } else {
+ this.bannerSince = null;
}
this.record.restAt = this.restCandidateSince ?? now;
this.record.finalStatus = status;
diff --git a/tests/restart-chaos/release-swap-schedule.test.ts b/tests/restart-chaos/release-swap-schedule.test.ts
index 9a028c52001..19266c280ba 100644
--- a/tests/restart-chaos/release-swap-schedule.test.ts
+++ b/tests/restart-chaos/release-swap-schedule.test.ts
@@ -32,6 +32,25 @@ describe("release swap options", () => {
expect(run.due(1_500_000)).toBe(run.plan.faults[1]);
});
+ it("drives a staging-mix workload's slots on the swap clock instead of a storm", () => {
+ const { args, releaseSwap } = expandReleaseSwapArgs(["--env", "daytona", "--workload", "staging-mix", "--release-swap", "--release-swap-count", "2",
+ "--release-swap-spacing-minutes", "5", "--release-swap-overlap-seconds", "120", "--release-swap-settle-minutes", "2"]);
+ expect(releaseSwap).toEqual({ swaps: 2, spacingMs: 300_000, overlapMs: 120_000, settleMs: 120_000 });
+ expect(args).toEqual(["--env", "daytona", "--workload", "staging-mix", "--mix-fault-spacing-minutes", "5", "--mix-minutes", "11",
+ "--faults", "K15", "--k15-overlap-seconds", "120", "--k15-fresh-home", "--ephemeral-server"]);
+ expect(() => expandReleaseSwapArgs(["--workload", "staging-mix", "--release-swap", "--mix-minutes", "30"])).toThrow(/--mix-minutes/);
+ const longest = ["--workload", "staging-mix", "--release-swap", "--release-swap-count", "6", "--release-swap-spacing-minutes"];
+ expect(() => expandReleaseSwapArgs([...longest, "100"])).toThrow(/--release-swap-count 6 × --release-swap-spacing-minutes 100 needs a 601-minute staging-mix wave; the longest is 600/);
+ expect(expandReleaseSwapArgs([...longest, "99"]).args).toEqual(expect.arrayContaining(["--mix-minutes", "595"]));
+ const command = parseCommand(["--env", "local", "--storage", "s3", "--workload", "staging-mix", "--mix-tasks", "6", "--mix-runs", "4", "--mix-queue", "2",
+ "--mix-turn-minutes", "5", "--seed", "7", "--release-swap", "--release-swap-count", "2", "--release-swap-spacing-minutes", "5",
+ "--release-swap-overlap-seconds", "120", "--release-swap-settle-minutes", "2"], 0);
+ if (command.mode !== "wave") throw new Error("expected a wave");
+ expect(command.options).toMatchObject({ faults: ["K15"], k15HoldMs: [120_000, 120_000], releaseSwap: { swaps: 2 },
+ stagingMix: { faultSpacingMinutes: 5, minutes: 11, runs: 4, queue: 2, tasks: 6 } });
+ expect(command.options.stormMinutes).toBeUndefined();
+ });
+
it("refuses flags the cell plans itself, stray shape flags, and an overlap the spacing cannot hold", () => {
expect(() => expandReleaseSwapArgs(["--release-swap", "--faults", "K1"])).toThrow(/--faults/);
expect(() => expandReleaseSwapArgs(["--release-swap-count", "2"])).toThrow(/add --release-swap/);
diff --git a/tests/restart-chaos/release-swap-schedule.ts b/tests/restart-chaos/release-swap-schedule.ts
index 179b94734e7..25fc20b7d0a 100644
--- a/tests/restart-chaos/release-swap-schedule.ts
+++ b/tests/restart-chaos/release-swap-schedule.ts
@@ -1,4 +1,5 @@
import type { StormFault, StormPlan } from "./types.js";
+import { MIX_MAX_MINUTES } from "./staging-mix.js";
export interface ReleaseSwapOptions {
swaps: number;
@@ -21,6 +22,8 @@ const FLAGS = {
"release-swap-drain-minutes": { key: "drainMs", unit: 60_000, min: 1, max: 240 },
} as const;
+const MIX_PLANNED = ["mix-fault-spacing-minutes", "mix-minutes"];
+
const REPLACED = ["storm-minutes", "faults", "k15-overlap-seconds", "k15-hold-seconds", "replay", "crash-point", "matrix", "upgrade-from"];
function flagName(arg: string): string | null {
@@ -73,8 +76,20 @@ export function expandReleaseSwapArgs(input: readonly string[]): { args: string[
}
const envIndex = rest.findIndex(arg => arg === "--env" || arg.startsWith("--env="));
const env = envIndex < 0 ? "local" : rest[envIndex]!.includes("=") ? rest[envIndex]!.split("=")[1] : rest[envIndex + 1];
- const added = ["--storm-minutes", String(Math.min(240, Math.ceil((options.swaps * options.spacingMs) / 60_000))), "--faults", "K15",
- "--k15-overlap-seconds", String(Math.round(options.overlapMs / 1000))];
+ const mix = given.has("workload");
+ if (mix) {
+ const planned = MIX_PLANNED.find(name => given.has(name));
+ if (planned) throw new Error(`--release-swap plans the staging-mix swap slots; do not combine it with --${planned}`);
+ }
+ const spacingMinutes = Math.ceil(options.spacingMs / 60_000);
+ if (mix && options.swaps * spacingMinutes + 1 > MIX_MAX_MINUTES) {
+ throw new Error(`--release-swap-count ${options.swaps} × --release-swap-spacing-minutes ${spacingMinutes} needs a ${options.swaps * spacingMinutes + 1}-minute `
+ + `staging-mix wave; the longest is ${MIX_MAX_MINUTES} minutes. Lower the swap count or the spacing.`);
+ }
+ const clock = mix
+ ? ["--mix-fault-spacing-minutes", String(spacingMinutes), "--mix-minutes", String(options.swaps * spacingMinutes + 1)]
+ : ["--storm-minutes", String(Math.min(240, Math.ceil((options.swaps * options.spacingMs) / 60_000)))];
+ const added = [...clock, "--faults", "K15", "--k15-overlap-seconds", String(Math.round(options.overlapMs / 1000))];
if (env === "daytona") {
if (!given.has("k15-fresh-home")) added.push("--k15-fresh-home");
if (!given.has("ephemeral-server")) added.push("--ephemeral-server");
diff --git a/tests/restart-chaos/release-swap-setup.ts b/tests/restart-chaos/release-swap-setup.ts
index 4fb96461abd..6d8a7e65c74 100644
--- a/tests/restart-chaos/release-swap-setup.ts
+++ b/tests/restart-chaos/release-swap-setup.ts
@@ -30,19 +30,21 @@ export async function diskUsageKb(directory: string): Promise {
}
export async function startReleaseSwap(input: { options: ReleaseSwapOptions; env: EnvironmentKind; adapters: readonly AdapterId[]; lab: ChaosLab;
- api: ChaosApi; db: Db; timeline: Timeline; temporaryRoot: string; freshHome: boolean; cancelled: () => boolean }): Promise {
+ api: ChaosApi; db: Db; timeline: Timeline; temporaryRoot: string; freshHome: boolean; cancelled: () => boolean }): Promise<{ driver: ReleaseSwapDriver; probesReady: Promise }> {
const shape = input.env === "daytona" ? DAYTONA_PROBE_SHAPE : LOCAL_PROBE_SHAPE;
const record = (kind: string, data: Record) => { input.timeline.add(kind, data); };
const agents = Object.fromEntries(input.adapters.flatMap(adapter => input.lab.agents[adapter] ? [[adapter, input.lab.agents[adapter]]] : []));
const probeAgents = await hireProbeAgents(input.api, input.lab.companyId, agents, shape.concurrency);
- await waitForAgentsReady(input.api, probeAgents, input.timeline, input.env === "daytona" ? 10 * 60_000 : undefined, 1_000, input.cancelled);
- record("release_swap_probe_agents", { agents: Object.fromEntries(Object.entries(probeAgents).map(([adapter, agent]) => [adapter, agent!.id])),
- maxConcurrentRuns: shape.concurrency, prepIntervalMs: shape.prepIntervalMs, prepProbeCap: shape.prepProbeCap });
+ const probesReady = waitForAgentsReady(input.api, probeAgents, input.timeline, input.env === "daytona" ? 10 * 60_000 : undefined, 1_000, input.cancelled).then(() => {
+ record("release_swap_probe_agents", { agents: Object.fromEntries(Object.entries(probeAgents).map(([adapter, agent]) => [adapter, agent!.id])),
+ maxConcurrentRuns: shape.concurrency, prepIntervalMs: shape.prepIntervalMs, prepProbeCap: shape.prepProbeCap });
+ });
+ probesReady.catch(() => undefined);
const driver = new ReleaseSwapDriver({ options: input.options, companyId: input.lab.companyId, adapters: input.adapters, agents: input.lab.agents,
probeAgents, prepIntervalMs: shape.prepIntervalMs, prepProbeCap: shape.prepProbeCap, reader: pgRunReader(input.db), record });
driver.plugins.push(homeCleanupPlugin({ temporaryRoot: input.temporaryRoot, freshHome: input.freshHome,
remove: directory => rm(directory, { recursive: true, force: true }), diskUsageKb, record }));
- return driver;
+ return { driver, probesReady };
}
export function stormRunFor(plan: WavePlan, now: number): StormRun | ReleaseSwapRun | null {
@@ -59,9 +61,9 @@ export class ReleaseSwapWave {
supervisor: Pick; cancelled: () => boolean }): Promise {
const options = input.plan.options;
const db = createDb(input.databaseUrl);
- const driver = await startReleaseSwap({ options: options.releaseSwap!, env: options.env, adapters: options.adapters, lab: input.lab, api: input.api, db,
+ const { driver, probesReady } = await startReleaseSwap({ options: options.releaseSwap!, env: options.env, adapters: options.adapters, lab: input.lab, api: input.api, db,
timeline: input.timeline, temporaryRoot: input.supervisor.paths.temporaryRoot, freshHome: options.k15FreshHome === true, cancelled: input.cancelled });
- input.supervisor.swapHook = driver.hook;
+ input.supervisor.swapHook = async info => { await probesReady; return driver.hook(info); };
setOverlapProbe(() => driver.launchOpen);
const wave = new ReleaseSwapWave(driver, db, input.plan, input.lab, input.api, input.timeline, input.supervisor, input.cancelled);
driver.plugins.push({ name: "overlap-launch-deadlines", onPhase: async (_swap, info) => { if (info.phase === "stopped") wave.extendOverlapLaunches(info.at); } });
diff --git a/tests/restart-chaos/server-freeze.test.ts b/tests/restart-chaos/server-freeze.test.ts
new file mode 100644
index 00000000000..963d8d692f6
--- /dev/null
+++ b/tests/restart-chaos/server-freeze.test.ts
@@ -0,0 +1,39 @@
+import { describe, expect, it } from "vitest";
+import { whileFrozen, type SendSignal } from "./server-freeze.js";
+
+function signals(exited = false) {
+ const sent: Array<[number, string | number]> = [];
+ const send: SendSignal = (pid, signal) => {
+ if (exited && signal === (0 as unknown as NodeJS.Signals)) throw Object.assign(new Error("no such process"), { code: "ESRCH" });
+ sent.push([pid, signal]);
+ };
+ return { sent, send };
+}
+
+describe("freezing the incoming server while the outgoing one stops", () => {
+ it("resumes the incoming server after the freeze when the stop succeeds", async () => {
+ const { sent, send } = signals();
+ const waits: number[] = [];
+ const frozen = await whileFrozen({ pid: 42, freezeMs: 150_000, send, wait: async ms => { waits.push(ms); } }, async () => "stopped");
+ expect(frozen.result).toBe("stopped");
+ expect(frozen.thawedAt).not.toBeNull();
+ expect(sent.filter(([, signal]) => signal !== 0)).toEqual([[42, "SIGSTOP"], [42, "SIGCONT"]]);
+ expect(waits[0]).toBeGreaterThan(149_000);
+ });
+
+ it("resumes the incoming server at once when stopping the outgoing one fails", async () => {
+ const { sent, send } = signals();
+ await expect(whileFrozen({ pid: 42, freezeMs: 150_000, send, wait: async () => undefined }, async () => { throw new Error("stop timed out"); }))
+ .rejects.toThrow(/stop timed out/);
+ expect(sent.filter(([, signal]) => signal !== 0)).toEqual([[42, "SIGSTOP"], [42, "SIGCONT"]]);
+ });
+
+ it("sends nothing after the incoming server exited, and nothing at all without a freeze", async () => {
+ const exited = signals(true);
+ await expect(whileFrozen({ pid: 42, freezeMs: 1_000, send: exited.send, wait: async () => undefined }, async () => { throw new Error("boom"); })).rejects.toThrow();
+ expect(exited.sent).toEqual([[42, "SIGSTOP"]]);
+ const none = signals();
+ expect(await whileFrozen({ pid: 42, freezeMs: null, send: none.send }, async () => 1)).toEqual({ result: 1, frozenAt: null, thawedAt: null });
+ expect(none.sent).toEqual([]);
+ });
+});
diff --git a/tests/restart-chaos/server-freeze.ts b/tests/restart-chaos/server-freeze.ts
new file mode 100644
index 00000000000..e2385363852
--- /dev/null
+++ b/tests/restart-chaos/server-freeze.ts
@@ -0,0 +1,31 @@
+import { setTimeout as delay } from "node:timers/promises";
+
+export type SendSignal = (pid: number, signal: NodeJS.Signals) => void;
+
+const sendSignal: SendSignal = (pid, signal) => { process.kill(pid, signal); };
+
+function alive(pid: number, send: SendSignal): boolean {
+ try { send(pid, 0 as unknown as NodeJS.Signals); return true; } catch { return false; }
+}
+
+export async function whileFrozen(input: { pid: number | null | undefined; freezeMs: number | null; send?: SendSignal; wait?: (ms: number) => Promise },
+ body: () => Promise): Promise<{ result: T; frozenAt: number | null; thawedAt: number | null }> {
+ const send = input.send ?? sendSignal;
+ const wait = input.wait ?? (ms => delay(ms));
+ const frozenAt = input.freezeMs !== null && input.pid ? Date.now() : null;
+ if (frozenAt !== null) send(input.pid!, "SIGSTOP");
+ let thawedAt: number | null = null;
+ const thaw = () => {
+ if (frozenAt === null || thawedAt !== null) return;
+ thawedAt = Date.now();
+ if (alive(input.pid!, send)) send(input.pid!, "SIGCONT");
+ };
+ try {
+ const result = await body();
+ if (frozenAt !== null) await wait(Math.max(0, frozenAt + input.freezeMs! - Date.now()));
+ thaw();
+ return { result, frozenAt, thawedAt };
+ } finally {
+ thaw();
+ }
+}
diff --git a/tests/restart-chaos/staging-mix-retry.test.ts b/tests/restart-chaos/staging-mix-retry.test.ts
index 8329327fd12..136314815c2 100644
--- a/tests/restart-chaos/staging-mix-retry.test.ts
+++ b/tests/restart-chaos/staging-mix-retry.test.ts
@@ -1,12 +1,12 @@
import { describe, expect, it } from "vitest";
import { parseCommand } from "./config.js";
import {
- collectStagingMixArtifacts, completedAtAnchor, doneCommentsBeforeCompletion, doneMarker, expectedRoundHash, gradeStagingMix, planStagingMixWave, ROUND_SECONDS,
+ collectStagingMixArtifacts, completedAtAnchor, completionRunIds, doneCommentsBeforeCompletion, doneMarker, expectedRoundHash, gradeStagingMix, planStagingMixWave, ROUND_SECONDS,
stagingMixDescription, type CompletionEvidence, type StagingMixTask,
} from "./staging-mix.js";
import {
- bannerSink, commentEvidence, evidenceCompletion, extraTurnLine, extraTurnsByReason, issueView, readAssignedWorkInvariant, retryAfterSettleStage, runsAfterDone, shouldClickRetryAfterSettle,
- stagingMixHeadline, summarizeStagingMix, type RetryStageObserver,
+ bannerSink, commentEvidence, evidenceCompletion, extraTurnLine, extraTurnsByReason, followUpLine, followUpsAnsweredLater, issueView, laterAnswerHeadline, readAssignedWorkInvariant,
+ retryAfterSettleStage, runsAfterDone, shouldClickRetryAfterSettle, stagingMixHeadline, summarizeStagingMix, type RetryStageObserver, type WakeEvidenceFile,
} from "./staging-mix-retry.js";
import type { OracleArtifacts, TaskPlan, TaskRecord, TaskSample } from "./types.js";
@@ -242,8 +242,8 @@ describe("staging-mix runs after the task completed", () => {
expect(neverStarted.map(turn => turn.runId)).toEqual(["aa11bb22-dddd"]);
expect(extraTurnLine(neverStarted[0]!)).toContain("never started");
expect(extraTurnsByReason(started)).toEqual({ issue_children_completed: 1, issue_recovery_action_restored: 1 });
- expect(runsAfterDone("t001", evidence, { at: null, source: null })).toEqual({ started: [], neverStarted: [] });
- expect(runsAfterDone("t001", null, { at: 1, source: "completed_at" })).toEqual({ started: [], neverStarted: [] });
+ expect(runsAfterDone("t001", evidence, { at: null, source: null })).toEqual({ started: [], neverStarted: [], answeredFollowUp: [] });
+ expect(runsAfterDone("t001", null, { at: 1, source: "completed_at" })).toEqual({ started: [], neverStarted: [], answeredFollowUp: [] });
});
it("does not count the run that completed the task as an extra turn (w-20261112T233630-s7201 t005)", () => {
@@ -264,6 +264,79 @@ describe("staging-mix runs after the task completed", () => {
});
});
+describe("a follow-up the running turn did not receive (w-20261113T052703-s7301 t010)", () => {
+ const nonce = "bd13d3bf";
+ const comment = "fd278a3a-946b-44df-b62f-0e47c982aea1";
+ const first = "0d18f0d9-0550-4f97-8926-0fcf49b43d19", second = "42acaf33-4c75-4d62-8bfe-07053a78cd9f";
+ const t010 = (): WakeEvidenceFile => ({
+ completedAt: 1_794_550_265_674,
+ runs: [{ runId: first, createdAt: 1_794_548_287_511, startedAt: 1_794_548_287_574, invocationSource: "assignment", triggerDetail: "system",
+ wakeupRequestId: "4816f3b7", acceptedDisposition: "done", acceptedAt: 1_794_549_206_395, contextSnapshot: { wakeReason: "issue_assigned", paperclipWake: { commentIds: [] } },
+ },
+ { runId: second, createdAt: 1_794_549_207_682, startedAt: 1_794_550_224_261, invocationSource: "automation", triggerDetail: "system",
+ wakeupRequestId: "452458ef", acceptedDisposition: "done", acceptedAt: 1_794_550_262_627,
+ contextSnapshot: { wakeReason: "issue_commented", wakeCommentIds: [comment], paperclipWake: { commentIds: [comment] } } }],
+ wakes: [{ id: "4816f3b7", reason: "issue_assigned", source: "assignment", triggerDetail: "system", requestedByActorType: "user", requestedByActorId: "local-board", createdAt: 1_794_548_287_511 },
+ { id: "452458ef", reason: "issue_execution_promoted", source: "automation", triggerDetail: "system", requestedByActorType: "user", requestedByActorId: "local-board",
+ createdAt: 1_794_548_961_691, payload: { commentId: comment, _paperclipWakeContext: { wakeCommentIds: [comment] } } }],
+ });
+ const posted = commentEvidence([
+ { body: "Done. The follow-up requirement is applied: the `ledger` document now includes `FOLLOWUP-bd13d3bf`; the single `DONE-bd13d3bf` comment was already posted.",
+ authorAgentId: "a", createdByRunId: second, createdAt: "2026-11-13T06:11:05.788Z" },
+ { body: "Task CHA-12 complete. No board comments arrived, so no extra lines were added.", authorAgentId: "a", createdByRunId: first, createdAt: "2026-11-13T05:53:27.608Z" },
+ { body: doneMarker(nonce), authorAgentId: "a", createdByRunId: first, createdAt: "2026-11-13T05:53:19.468Z" }]);
+ const completionOnce = (evidence: WakeEvidenceFile) => gradeStagingMix(nonce, { turnMinutes: 15, rounds: 9, subtasks: 0, followUpAfterMs: 400_000, arriveAtMs: 0, holdMs: 0 },
+ { issueStatus: "done", documents: { ledger: ledger(nonce, 9) }, agentComments: [], files: {}, commands: {}, workspacePath: null,
+ stagingMix: { followUp: null, followUpErrors: [], retries: [], comments: posted, completion: evidenceCompletion(evidence)! } }).checks.find(check => check.name === "completion_once")!;
+ const afterDone = (evidence: WakeEvidenceFile) => runsAfterDone("t010", evidence, completedAtAnchor(evidenceCompletion(evidence), posted, nonce));
+
+ it("counts the run that answered the unreceived comment as required, not as an extra turn or a duplicate completion", () => {
+ const evidence = t010();
+ expect(afterDone(evidence).started).toEqual([]);
+ expect(afterDone(evidence).answeredFollowUp.map(turn => turn.runId)).toEqual([second]);
+ expect(completionOnce(evidence)).toMatchObject({ ok: true, detail: "runs=1" });
+ const answered = followUpsAnsweredLater("t010", evidence);
+ expect(answered).toEqual([{ taskId: "t010", runId: second, commentIds: [comment], postedAt: 1_794_548_961_691, queuedAt: 1_794_549_207_682, startedAt: 1_794_550_224_261 }]);
+ expect(followUpLine(answered[0]!)).toBe("- t010: run 42acaf33 answered fd278a3a 1263 s after it was posted, after 1017 s queued");
+ expect(laterAnswerHeadline(0, answered)).toEqual(["extra turns (a run started after completion with no undelivered input): 0",
+ "follow-up answered in a later run: 1, queued wait p50 1017 s / max 1017 s, posted to answered p50 1263 s / max 1263 s"]);
+ });
+
+ const withRun = (base: WakeEvidenceFile, runId: string, wake: { id: string; reason: string; payload?: unknown }, at: number): WakeEvidenceFile => {
+ const run2 = t010().runs![1]!;
+ return { ...base, runs: [...base.runs!, { ...run2, runId, createdAt: at, startedAt: at + 900, wakeupRequestId: wake.id, contextSnapshot: { wakeReason: wake.reason } }],
+ wakes: [...base.wakes!, { id: wake.id, reason: wake.reason, source: "automation", triggerDetail: "system", requestedByActorType: "system", requestedByActorId: null,
+ createdAt: at, payload: wake.payload ?? {} }] };
+ };
+
+ it("treats the server's re-dispatch of an owed comment after a failed attempt as required", () => {
+ const base = t010();
+ const failed = { ...base, runs: [base.runs![0]!, { ...base.runs![1]!, status: "failed", acceptedDisposition: null, acceptedAt: null }] };
+ const evidence = withRun(failed, "9e9e9e9e-retry", { id: "retry-wake", reason: "process_lost_retry", payload: { _paperclipWakeContext: { wakeCommentIds: [comment] } } },
+ 1_794_550_230_000);
+ expect(afterDone(evidence).started).toEqual([]);
+ expect(afterDone(evidence).answeredFollowUp.map(turn => turn.runId)).toEqual([second, "9e9e9e9e-retry"]);
+ expect(completionOnce(evidence)).toMatchObject({ ok: true, detail: "runs=1" });
+ });
+
+ it("counts a run started after completion from a wake that carries no comment as an extra turn and a duplicate completion", () => {
+ const evidence = withRun(t010(), "7a7a7a7a-extra", { id: "recovery-wake", reason: "issue_recovery_action_restored" }, 1_794_550_300_000);
+ expect(afterDone(evidence).started.map(turn => turn.runId)).toEqual(["7a7a7a7a-extra"]);
+ expect(afterDone(evidence).answeredFollowUp.map(turn => turn.runId)).toEqual([second]);
+ expect(completionOnce(evidence)).toMatchObject({ ok: false, detail: "duplicate completion: runs 0d18f0d9, 7a7a7a7a each reported completion" });
+ });
+
+ it("lists nothing for a comment wake the server cancelled as already delivered, and an extra run for the same comment is still extra", () => {
+ const base = t010();
+ const cancelled: WakeEvidenceFile = { ...base, runs: [base.runs![0]!] };
+ expect(followUpsAnsweredLater("t010", cancelled)).toEqual([]);
+ expect(afterDone(cancelled)).toEqual({ started: [], neverStarted: [], answeredFollowUp: [] });
+ const replayed = withRun(cancelled, "5b5b5b5b-replay", { id: "replay-wake", reason: "issue_recovery_action_restored" }, 1_794_550_300_000);
+ expect(afterDone(replayed).started.map(turn => turn.runId)).toEqual(["5b5b5b5b-replay"]);
+ expect(completionOnce(replayed).ok).toBe(false);
+ });
+});
+
describe("staging-mix completion from accepted results", () => {
const nonce = "abcd1234";
const completion = (runs: CompletionEvidence["runs"]): CompletionEvidence => ({ completedAt: 10_000, runs });
@@ -283,6 +356,31 @@ describe("staging-mix completion from accepted results", () => {
stagingMix: { followUp: null, followUpErrors: [], retries: [], comments: posted, completion: twice } }).checks.find(check => check.name === "completion_once")!.ok).toBe(false);
});
+ type Row = { runId: string; at: number; disposition: string | null; followUp?: boolean };
+ const first = (disposition: string | null): Row => ({ runId: "first", at: 1_000, disposition });
+ const followUp = (at: number, disposition: string | null): Row => ({ runId: `follow-up-${at}`, at, disposition, followUp: true });
+ const extra = (at: number): Row => ({ runId: `extra-${at}`, at, disposition: "done" });
+ it.each<{ name: string; rows: Row[]; ok: boolean; runs: string[] }>([
+ { name: "first completion by the first run", rows: [first("done")], ok: true, runs: ["first"] },
+ { name: "first completion by a follow-up run after a yielded first run", rows: [first("yielded"), followUp(2_000, "done")], ok: true, runs: ["follow-up-2000"] },
+ { name: "first completion by the retry of a follow-up run that failed before its turn", rows: [first("yielded"), followUp(2_000, null), followUp(3_000, "done")],
+ ok: true, runs: ["follow-up-3000"] },
+ { name: "a later completion that answers new input", rows: [first("done"), followUp(2_000, "done")], ok: true, runs: ["first"] },
+ { name: "a later completion that answers nothing", rows: [first("done"), extra(2_000)], ok: false, runs: ["first", "extra-2000"] },
+ { name: "a follow-up run completes first, then a run that answers nothing completes again", rows: [first("yielded"), followUp(2_000, "done"), extra(3_000)],
+ ok: false, runs: ["follow-up-2000", "extra-3000"] },
+ { name: "a later follow-up completion, then a run that answers nothing", rows: [first("done"), followUp(2_000, "done"), extra(3_000)],
+ ok: false, runs: ["first", "extra-3000"] },
+ ])("completion_once truth table: $name", ({ rows, ok, runs }) => {
+ const evidence = completion(rows.map(row => ({ runId: row.runId, startedAt: row.at - 500, acceptedDisposition: row.disposition,
+ acceptedAt: row.disposition ? row.at : null, answersFollowUp: row.followUp ?? false })));
+ expect(completionRunIds([], nonce, evidence)).toEqual(runs);
+ const check = gradeStagingMix(nonce, { turnMinutes: 5, rounds: 4, subtasks: 0, followUpAfterMs: null, arriveAtMs: 0, holdMs: 0 },
+ { issueStatus: "done", documents: { ledger: ledger(nonce, 4) }, agentComments: [], files: {}, commands: {}, workspacePath: null,
+ stagingMix: { followUp: null, followUpErrors: [], retries: [], comments: [], completion: evidence } }).checks.find(entry => entry.name === "completion_once")!;
+ expect(check.ok).toBe(ok);
+ });
+
it("reports a DONE comment from a run whose accepted result was not done as its own finding", () => {
const runs = completion([{ runId: "r1", startedAt: 1, acceptedDisposition: "blocked", acceptedAt: 5_000 }, { runId: "r2", startedAt: 6_000, acceptedDisposition: "done", acceptedAt: 9_000 }]);
expect(doneCommentsBeforeCompletion([{ body: doneMarker(nonce), runId: "r1", createdAt: 4_900 }, { body: doneMarker(nonce), runId: "r2", createdAt: 8_900 }], nonce, runs))
diff --git a/tests/restart-chaos/staging-mix-retry.ts b/tests/restart-chaos/staging-mix-retry.ts
index cea5753242c..1b52d73dd5f 100644
--- a/tests/restart-chaos/staging-mix-retry.ts
+++ b/tests/restart-chaos/staging-mix-retry.ts
@@ -6,6 +6,7 @@ import {
bannerShownMs, completedAtAnchor, doneCommentsBeforeCompletion, DUPLICATE_CHECKS, firstDoneCommentAt, noteRetryBanner, type CompletionEvidence,
} from "./staging-mix.js";
import type { InterruptionSample, OracleResult, TaskPlan, TaskRecord, TaskSample } from "./types.js";
+import { laterAnswers, laterAnswerWaits, type LaterAnswer } from "./follow-up-delivery.js";
type Json = Record;
@@ -191,10 +192,13 @@ export interface ExtraTurn {
export interface WakeEvidenceFile {
completedAt?: number | null;
runs?: Array<{ runId: string | null; createdAt: number | null; startedAt?: number | null; invocationSource: string | null; triggerDetail: string | null;
- wakeupRequestId: string | null; acceptedDisposition?: string | null; acceptedAt?: number | null }>;
- wakes?: Array<{ id: string | null; reason: string | null; source: string | null; triggerDetail: string | null; requestedByActorType: string | null; requestedByActorId: string | null }>;
+ wakeupRequestId: string | null; acceptedDisposition?: string | null; acceptedAt?: number | null; finishedAt?: number | null; contextSnapshot?: unknown; status?: string | null }>;
+ wakes?: Array<{ id: string | null; reason: string | null; source: string | null; triggerDetail: string | null; requestedByActorType: string | null; requestedByActorId: string | null;
+ createdAt?: number | null; payload?: unknown }>;
}
+export interface FollowUpAnsweredLater extends LaterAnswer { taskId: string }
+
export type CommentRow = { body?: unknown; authorAgentId?: unknown; createdByRunId?: unknown; createdAt?: unknown };
export function commentEvidence(rows: readonly CommentRow[] | null): Array<{ body: string; runId: string | null; createdAt: number | null }> {
@@ -207,14 +211,16 @@ export function commentEvidence(rows: readonly CommentRow[] | null): Array<{ bod
export function evidenceCompletion(evidence: WakeEvidenceFile | null, issueCompletedAt?: unknown): CompletionEvidence | null {
const fromIssue = typeof issueCompletedAt === "string" && Number.isFinite(Date.parse(issueCompletedAt)) ? Date.parse(issueCompletedAt) : null;
if (!evidence && fromIssue === null) return null;
+ const answers = laterAnswers(evidence);
return { completedAt: evidence?.completedAt ?? fromIssue, runs: (evidence?.runs ?? []).map(run => ({ runId: run.runId, startedAt: run.startedAt ?? null,
- acceptedDisposition: run.acceptedDisposition ?? null, acceptedAt: run.acceptedAt ?? null })) };
+ acceptedDisposition: run.acceptedDisposition ?? null, acceptedAt: run.acceptedAt ?? null, answersFollowUp: Boolean(run.runId && answers.has(run.runId)) })) };
}
export function runsAfterDone(taskId: string, evidence: WakeEvidenceFile | null, anchor: { at: number | null; source: string | null }, doneCommentAt: number | null = null):
- { started: ExtraTurn[]; neverStarted: ExtraTurn[] } {
+ { started: ExtraTurn[]; neverStarted: ExtraTurn[]; answeredFollowUp: ExtraTurn[] } {
const doneAt = anchor.at;
- if (!evidence || doneAt === null) return { started: [], neverStarted: [] };
+ if (!evidence || doneAt === null) return { started: [], neverStarted: [], answeredFollowUp: [] };
+ const answers = laterAnswers(evidence);
const wakes = new Map((evidence.wakes ?? []).map(wake => [wake.id, wake]));
const turn = (run: NonNullable[number]): ExtraTurn => {
const wake = run.wakeupRequestId ? wakes.get(run.wakeupRequestId) : undefined;
@@ -223,12 +229,33 @@ export function runsAfterDone(taskId: string, evidence: WakeEvidenceFile | null,
actorType: wake?.requestedByActorType ?? null, actorId: wake?.requestedByActorId ?? null };
};
const runs = evidence.runs ?? [];
+ const afterDone = runs.filter(run => run.startedAt != null && run.startedAt > doneAt);
+ const answered = (run: (typeof runs)[number]) => Boolean(run.runId && answers.has(run.runId));
return {
- started: runs.filter(run => run.startedAt != null && run.startedAt > doneAt).map(turn),
+ started: afterDone.filter(run => !answered(run)).map(turn),
neverStarted: runs.filter(run => run.startedAt == null && run.createdAt !== null && run.createdAt > doneAt).map(turn),
+ answeredFollowUp: afterDone.filter(answered).map(turn),
};
}
+export function followUpsAnsweredLater(taskId: string, evidence: WakeEvidenceFile | null): FollowUpAnsweredLater[] {
+ return [...laterAnswers(evidence).values()].map(answer => ({ taskId, ...answer }));
+}
+
+const seconds = (value: number | null) => value === null ? "?" : `${Math.round(value / 1000)} s`;
+
+export function followUpLine(answer: FollowUpAnsweredLater): string {
+ return `- ${answer.taskId}: run ${answer.runId.slice(0, 8)} answered ${answer.commentIds.map(id => id.slice(0, 8)).join(", ")} `
+ + `${seconds(answer.postedAt === null ? null : answer.startedAt - answer.postedAt)} after it was posted, after ${seconds(answer.queuedAt === null ? null : answer.startedAt - answer.queuedAt)} queued`;
+}
+
+export function laterAnswerHeadline(extraTurns: number, answers: readonly FollowUpAnsweredLater[]): string[] {
+ const waits = laterAnswerWaits(answers);
+ return [`extra turns (a run started after completion with no undelivered input): ${extraTurns}`,
+ `follow-up answered in a later run: ${waits.count}${waits.count ? `, queued wait p50 ${seconds(waits.queueWaitP50Ms)} / max ${seconds(waits.queueWaitMaxMs)}, `
+ + `posted to answered p50 ${seconds(waits.answerAfterPostP50Ms)} / max ${seconds(waits.answerAfterPostMaxMs)}` : ""}`];
+}
+
const iso = (at: number | null) => at === null ? "?" : new Date(at).toISOString();
export function extraTurnLine(turn: ExtraTurn): string {
@@ -264,6 +291,7 @@ export async function writeStagingMixResults(input: { waveDir: string; api: Pick
const tasks: ResultTask[] = [];
const extraTurns: ExtraTurn[] = [];
const queuedAfterDone: ExtraTurn[] = [];
+ const answeredLater: FollowUpAnsweredLater[] = [];
const doneBeforeCompletion: Array<{ taskId: string; runId: string; disposition: string }> = [];
for (const task of input.tasks) {
const issue = task.record.issueId ? await input.api.get(`/api/issues/${task.record.issueId}`, 10_000).catch(() => null) : null;
@@ -276,6 +304,7 @@ export async function writeStagingMixResults(input: { waveDir: string; api: Pick
doneBeforeCompletion.push(...doneCommentsBeforeCompletion(posted, task.plan.nonce, completion).map(entry => ({ taskId: task.plan.taskId, ...entry })));
extraTurns.push(...afterDone.started);
queuedAfterDone.push(...afterDone.neverStarted);
+ answeredLater.push(...followUpsAnsweredLater(task.plan.taskId, evidence));
const grade = await readFile(path.join(input.waveDir, `${task.plan.taskId}.grade.json`), "utf8")
.then(text => JSON.parse(text) as { grade?: { T?: OracleResult } }).catch(() => null);
tasks.push({ taskId: task.plan.taskId, issueId: task.record.issueId, record: task.record, finalStatus: issue?.status ?? task.record.finalStatus,
@@ -283,16 +312,19 @@ export async function writeStagingMixResults(input: { waveDir: string; api: Pick
}
const result = summarizeStagingMix(tasks, extraClicks, await readAssignedWorkInvariant(input.api, input.companyId));
await writeFile(path.join(input.waveDir, "staging-mix-results.json"), `${JSON.stringify({ schema: "paperclip.restart-chaos.staging-mix/v1",
- headline: stagingMixHeadline(result), ...result, extraTurns, extraTurnsByReason: extraTurnsByReason(extraTurns), queuedAfterDone, doneBeforeCompletion }, null, 2)}\n`, { mode: 0o600 });
+ headline: [...stagingMixHeadline(result), ...laterAnswerHeadline(extraTurns.length, answeredLater)], ...result, extraTurns, extraTurnsByReason: extraTurnsByReason(extraTurns),
+ queuedAfterDone, doneBeforeCompletion, followUpsAnsweredLater: answeredLater, followUpWaits: laterAnswerWaits(answeredLater) }, null, 2)}\n`, { mode: 0o600 });
const invariant = result.invariant.violationCount === null ? `unreadable${result.invariant.error ? ` (${result.invariant.error})` : ""}`
: `${result.invariant.violationCount} violations, ${result.invariant.agentsWithStalledSlots} agents with stalled slots, ${result.invariant.openTasks} open tasks`;
- await writeFile(path.join(input.waveDir, "staging-mix-summary.md"), [...stagingMixHeadline(result), `Assigned-work invariant: ${invariant}`, "",
+ await writeFile(path.join(input.waveDir, "staging-mix-summary.md"), [...stagingMixHeadline(result), ...laterAnswerHeadline(extraTurns.length, answeredLater),
+ `Assigned-work invariant: ${invariant}`, "",
...result.duplicatedSideEffects.map(entry => `- ${entry.taskId}: duplicated ${entry.checks.join(", ")}`),
...result.evidenceIncomplete.map(entry => `- ${entry.taskId}: evidence incomplete for ${entry.checks.join(", ")}`),
...result.bannerShown.map(entry => `- ${entry.taskId}: Retry banner from ${entry.firstShownAt} for ${Math.round(entry.shownMs / 1000)} s`
+ `${entry.errorCode ? ` (${entry.errorCode})` : ""}${entry.completedAnyway ? ", completed without a click" : ""}`),
...(extraTurns.length ? ["", `Runs started after the task completed: ${extraTurns.length}`, ...extraTurns.map(extraTurnLine)] : []),
...(queuedAfterDone.length ? ["", `Runs queued after the task completed, never started: ${queuedAfterDone.length}`, ...queuedAfterDone.map(extraTurnLine)] : []),
+ ...(answeredLater.length ? ["", `Follow-ups the running turn did not receive, answered in a later run: ${answeredLater.length}`, ...answeredLater.map(followUpLine)] : []),
...(doneBeforeCompletion.length ? ["", `DONE comment posted before the task completed: ${doneBeforeCompletion.length}`,
...doneBeforeCompletion.map(entry => `- ${entry.taskId}: run ${entry.runId.slice(0, 8)} posted DONE but its accepted result was ${entry.disposition}`)] : []), ""].join("\n"), { mode: 0o600 });
return result;
diff --git a/tests/restart-chaos/staging-mix.test.ts b/tests/restart-chaos/staging-mix.test.ts
index 7dcfd30df49..c6c82ef5cbc 100644
--- a/tests/restart-chaos/staging-mix.test.ts
+++ b/tests/restart-chaos/staging-mix.test.ts
@@ -4,7 +4,7 @@ import { parseCommand } from "./config.js";
import {
agentRunCaps, assignAdapters, childLine, childNonce, doneMarker, expectedRoundHash, followUpMarker, gradeStagingMix, holdsRetry, planArrivals,
arrivalEndMs, followUpRequestId, planStagingMixWave, plannedTaskCount, posixCksum, retryDue, roundCommand, ROUND_SECONDS, setOverlapProbe, stagingMixDescription,
- stagingMixBudgetMs, stagingMixStep, SUBTASK_ROUNDS, verifyLedger, type StagingMixTask,
+ slotTimesMs, stagingMixBudgetMs, stagingMixStep, SUBTASK_ROUNDS, verifyLedger, type StagingMixTask,
} from "./staging-mix.js";
import { descendantsOf, parseMemInfo, parsePs, parseVmStat, StagingMixRun, stagingMixSeedOptions, type ResourceSample } from "./staging-mix-run.js";
import type { OracleArtifacts, TaskRecord, TaskSample } from "./types.js";
@@ -115,6 +115,14 @@ describe("staging-mix options", () => {
});
describe("staging-mix planning", () => {
+ it("moves the first fault slot early without adding a slot (release swap: 2 swaps 4 minutes apart)", () => {
+ expect(slotTimesMs({ minutes: 9, faultSpacingMinutes: 4 }, undefined)).toEqual([240_000, 480_000]);
+ expect(slotTimesMs({ minutes: 9, faultSpacingMinutes: 4 }, "30")).toEqual([30_000, 270_000]);
+ expect(slotTimesMs({ minutes: 29, faultSpacingMinutes: 14 }, "60")).toEqual([60_000, 900_000]);
+ expect(slotTimesMs({ minutes: 28, faultSpacingMinutes: 14 }, "60")).toEqual([60_000]);
+ expect(slotTimesMs({ minutes: 3, faultSpacingMinutes: 4 }, "30")).toEqual([]);
+ });
+
it("interleaves adapters so every prefix keeps the ratio", () => {
const adapters = assignAdapters([["opencode", 60], ["codex", 25], ["claude", 15]], 100);
const count = (slice: string[], adapter: string) => slice.filter(entry => entry === adapter).length;
diff --git a/tests/restart-chaos/staging-mix.ts b/tests/restart-chaos/staging-mix.ts
index b84536c645a..649e9667fb0 100644
--- a/tests/restart-chaos/staging-mix.ts
+++ b/tests/restart-chaos/staging-mix.ts
@@ -5,6 +5,7 @@ import type { AdapterId, FaultKind, OracleArtifacts, OracleResult, TaskFaultKind
import type { WorkloadAgents, WorkloadDefinition } from "./workloads.js";
export const STAGING_MIX = "staging-mix";
+export const MIX_MAX_MINUTES = 600;
export const ROUND_TICKS = 45;
export const ROUND_SECONDS = ROUND_TICKS * 2;
export const SUBTASKS_PER_PARENT = 2;
@@ -60,7 +61,7 @@ export interface StagingMixRecord {
export interface CompletionEvidence {
completedAt: number | null;
- runs: Array<{ runId: string | null; startedAt: number | null; acceptedDisposition: string | null; acceptedAt: number | null }>;
+ runs: Array<{ runId: string | null; startedAt: number | null; acceptedDisposition: string | null; acceptedAt: number | null; answersFollowUp?: boolean }>;
}
export interface RetryBanner { firstShownAt: number; lastShownAt: number; shownMs: number; visibleSince: number | null; errorCode: string | null }
@@ -145,7 +146,7 @@ export function applyStagingMix(options: WaveOptions, values: MixArgs, environme
throw new Error(`--workload ${STAGING_MIX} takes restart faults only`);
}
const turnMinutes = parseRange(String(values["mix-turn-minutes"] ?? "15-45"), "mix-turn-minutes", 1, 60);
- const minutes = whole(values["mix-minutes"], "mix-minutes", 240, 1, 600);
+ const minutes = whole(values["mix-minutes"], "mix-minutes", 240, 1, MIX_MAX_MINUTES);
const faultSpacingMinutes = whole(values["mix-fault-spacing-minutes"], "mix-fault-spacing-minutes", 40, 1, 600);
const arrivalCutoffMinutes = whole(values["mix-arrival-cutoff-minutes"], "mix-arrival-cutoff-minutes", turnMinutes[1], 0, 600);
const runCapDefault = whole(environment.RESTART_CHAOS_MIX_RUN_CAP, "RESTART_CHAOS_MIX_RUN_CAP", 30, 1, 200);
@@ -220,10 +221,12 @@ function meanCycleMs(mix: Pick): number {
return ((mix.turnMinutes[0] + mix.turnMinutes[1]) / 2 + 5) * 60_000;
}
-export function slotTimesMs(mix: Pick): number[] {
- const times: number[] = [];
- for (let at = mix.faultSpacingMinutes * 60_000; at < mix.minutes * 60_000; at += mix.faultSpacingMinutes * 60_000) times.push(at);
- return times;
+export function slotTimesMs(mix: Pick,
+ firstSlotRaw = process.env.RESTART_CHAOS_MIX_FIRST_SLOT_SECONDS): number[] {
+ const spacing = mix.faultSpacingMinutes * 60_000;
+ const first = firstSlotRaw && /^[0-9]+$/.test(firstSlotRaw) ? Math.min(Number(firstSlotRaw) * 1000, spacing) : spacing;
+ const count = Math.max(0, Math.ceil((mix.minutes * 60_000 - spacing) / spacing));
+ return Array.from({ length: count }, (_, index) => first + index * spacing);
}
export function maxSlotHoldMs(options: Pick): number {
@@ -409,8 +412,16 @@ export function completionRunIds(comments: readonly CommentEvidence[], nonce: st
const summaries = comments.filter(comment => anchor !== null && typeof comment.createdAt === "number" && comment.createdAt >= anchor
&& comment.body.includes(doneMarker(nonce)) && comment.runId).map(comment => comment.runId!);
if (hasDispositions(completion)) {
+ const byId = new Map(completion.runs.flatMap(run => run.runId ? [[run.runId, run] as const] : []));
+ const reportedAt = (runId: string) => {
+ const run = byId.get(runId);
+ if (run?.acceptedDisposition === "done" && run.acceptedAt !== null) return run.acceptedAt;
+ const posted = comments.filter(comment => comment.runId === runId && typeof comment.createdAt === "number").map(comment => comment.createdAt!);
+ return posted.length ? Math.min(...posted) : run?.startedAt ?? Number.POSITIVE_INFINITY;
+ };
const accepted = completion.runs.filter(run => run.acceptedDisposition === "done" && run.runId).map(run => run.runId!);
- return [...new Set([...accepted, ...summaries])];
+ const ordered = [...new Set([...accepted, ...summaries])].sort((a, b) => reportedAt(a) - reportedAt(b));
+ return ordered.filter((runId, index) => index === 0 || !byId.get(runId)?.answersFollowUp);
}
return [...new Set([...comments.filter(comment => isDoneComment(comment.body, nonce) && comment.runId).map(comment => comment.runId!), ...summaries])];
}
diff --git a/tests/restart-chaos/supervisor.ts b/tests/restart-chaos/supervisor.ts
index 99c6205132b..61062e49628 100644
--- a/tests/restart-chaos/supervisor.ts
+++ b/tests/restart-chaos/supervisor.ts
@@ -42,6 +42,7 @@ import { isMultiServerFault, runMultiServerFault, type MultiServerKind, type Mul
import { generationHome, nextRedeployIdentity, redeployEnvironment, redeployIdentity, redeployNodeArgs, redeployReceipt, type RedeployIdentity } from "./redeploy.js";
import { startEdgeProxy, type EdgeProxy } from "./edge-proxy.js";
import { ephemeralServerEnvironment } from "./ephemeral-server.js";
+import { whileFrozen } from "./server-freeze.js";
import type { SwapPhase, SwapPhaseInfo } from "./release-swap-driver.js";
import { CLASS_R_PRE_FAULT_DEADLINE_MS, classRRedeploys, isClassRFault, runPluginStall, withReplacementPluginStall, type ClassRHooks } from "./class-r.js";
@@ -74,6 +75,12 @@ export function crashLoopShouldFire(target: (typeof CRASH_LOOP_TARGETS)[number],
return phase === "ready";
}
+export function overlapKillAfterMs(raw: string | undefined, overlapMs: number, name = "RESTART_CHAOS_K15_KILL_OLD_AFTER_SECONDS"): number | null {
+ if (raw === undefined || raw === "") return null;
+ if (!/^[0-9]+$/.test(raw)) throw new Error(`${name} must be a whole number of seconds`);
+ return Math.min(Number(raw) * 1000, overlapMs);
+}
+
/**
* Agents reach the server at PAPERCLIP_API_URL. An explicit value wins over the URL the server
* derives from its public URL (a developer's .paperclip/.env or shell can name a tailnet origin
@@ -176,6 +183,7 @@ export class ChaosSupervisor {
deployShape: { overlap: boolean; freshHome: boolean } = { overlap: false, freshHome: false };
ephemeralServer = false;
swapHook: ((info: SwapPhaseInfo) => Promise) | null = null;
+ private rollingRedeploys = 0;
/** `--k15-fresh-home`: the instance home and config of the serving generation (generation 0 uses `paths`). */
private serverHome: { home: string; configPath: string };
/** `--k15-overlap-seconds`: the edge on the wave's port, and the serving generation's own port behind it. */
@@ -822,10 +830,29 @@ export class ChaosSupervisor {
this.edge.target = this.backendPort;
this.timeline.add("k15_traffic_switched", { faultId: receipt.faultId, fromPort, toPort: this.backendPort, overlapMs });
await swap("switched", next);
- for (const until = Date.now() + overlapMs; Date.now() < until && !this.shuttingDown;) await delay(Math.min(1_000, until - Date.now()));
- await swap("stopping", next); await this.stopper(previous, "SIGTERM");
- receipt.stoppedAt = Date.now();
- this.timeline.add("fault_server_down", { faultId: receipt.faultId, previousPid: previous.pid, overlapMs });
+ this.rollingRedeploys += 1;
+ const killSwaps = process.env.RESTART_CHAOS_K15_KILL_OLD_SWAPS?.split(",").map(Number) ?? null;
+ const killAfterMs = killSwaps && !killSwaps.includes(this.rollingRedeploys) ? null
+ : overlapKillAfterMs(process.env.RESTART_CHAOS_K15_KILL_OLD_AFTER_SECONDS, overlapMs);
+ for (const until = Date.now() + (killAfterMs ?? overlapMs); Date.now() < until && !this.shuttingDown;) await delay(Math.min(1_000, until - Date.now()));
+ await swap("stopping", next);
+ const freezeSwaps = process.env.RESTART_CHAOS_K15_FREEZE_SWAPS?.split(",").map(Number) ?? null;
+ const freezeMs = freezeSwaps && !freezeSwaps.includes(this.rollingRedeploys) ? null
+ : overlapKillAfterMs(process.env.RESTART_CHAOS_K15_FREEZE_NEW_SECONDS, 600_000, "RESTART_CHAOS_K15_FREEZE_NEW_SECONDS");
+ if (freezeMs !== null && next.pid) this.timeline.add("k15_new_server_frozen", { faultId: receipt.faultId, pid: next.pid, forMs: freezeMs });
+ const frozen = await whileFrozen({ pid: next.pid, freezeMs }, async () => {
+ if (killAfterMs === null) await this.stopper(previous, "SIGTERM");
+ else {
+ this.timeline.add("k15_old_server_killed", { faultId: receipt.faultId, previousPid: previous.pid, afterMs: killAfterMs });
+ await this.stopper.crashForRestart(previous);
+ }
+ receipt.stoppedAt = Date.now();
+ this.timeline.add("fault_server_down", { faultId: receipt.faultId, previousPid: previous.pid, overlapMs });
+ });
+ if (frozen.frozenAt !== null) {
+ this.timeline.add("k15_new_server_thawed", { faultId: receipt.faultId, pid: next.pid, frozenMs: frozen.thawedAt! - frozen.frozenAt,
+ afterOldExitMs: frozen.thawedAt! - receipt.stoppedAt! });
+ }
await swap("stopped", next);
await this.redeployEvidence?.(receipt.faultId, "stopped");
}
diff --git a/tests/restart-chaos/ttft-baseline.ts b/tests/restart-chaos/ttft-baseline.ts
index 7ac5281f74f..aa81bd08bf6 100644
--- a/tests/restart-chaos/ttft-baseline.ts
+++ b/tests/restart-chaos/ttft-baseline.ts
@@ -331,8 +331,8 @@ async function main() {
const cleanup = await owned.cleanUp();
timeline.add("daytona_cleanup", cleanup);
await writeFile(path.join(outDir, "sandbox-cleanup.json"), `${JSON.stringify(cleanup, null, 2)}\n`, { mode: 0o600 });
- if (cleanup.remainingSandboxes.length || cleanup.errors.length || cleanup.environmentDeleteBlocked || !cleanup.environmentDeleted) {
- process.stderr.write(`Daytona cleanup incomplete: remaining ${cleanup.remainingSandboxes.join(", ") || "none"}; `
+ if (cleanup.remainingSandboxes.length || cleanup.undeletedSandboxes.length || cleanup.errors.length || cleanup.environmentDeleteBlocked || !cleanup.environmentDeleted) {
+ process.stderr.write(`Daytona cleanup incomplete: remaining ${cleanup.remainingSandboxes.join(", ") || "none"}; left after deletion ${cleanup.undeletedSandboxes.join(", ") || "none"}; `
+ `environment deleted ${cleanup.environmentDeleted}${cleanup.environmentDeleteBlocked ? ` (blocked: ${String(cleanup.environmentDeleteBlocked)})` : ""}; ${cleanup.errors.join("; ")}\n`);
process.exitCode = 1;
}
diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts
index 1e528be26cb..a9cb8c61f2a 100644
--- a/tests/restart-chaos/types.ts
+++ b/tests/restart-chaos/types.ts
@@ -294,6 +294,7 @@ export interface TaskSample {
recoveryOwner?: string | null;
executionBlocker?: boolean;
scheduledRetry?: boolean;
+ waitingOnChildren?: boolean;
runs?: RunSample[];
}
diff --git a/tests/restart-chaos/universal.test.ts b/tests/restart-chaos/universal.test.ts
index 84ad5eb3093..ee85e466646 100644
--- a/tests/restart-chaos/universal.test.ts
+++ b/tests/restart-chaos/universal.test.ts
@@ -1,7 +1,9 @@
import { describe, expect, it } from "vitest";
import { gradeTask } from "./grader.js";
import type { OracleArtifacts, RunSample, TaskRecord, TaskSample } from "./types.js";
-import { bareCauseMessage, DEFAULT_UNIVERSAL_BOUNDS as BOUNDS, diagnosableFailureCause, gradeU1, gradeU2, gradeU3, gradeU4, gradeU5, gradeUniversal, universalBounds } from "./universal.js";
+import { bareCauseMessage, declaredQueueBoundMs, DEFAULT_UNIVERSAL_BOUNDS as BOUNDS, diagnosableFailureCause, gradeU1, gradeU2, gradeU3, gradeU4, gradeU5, gradeUniversal,
+ universalBounds } from "./universal.js";
+import { slotFreeFrom } from "./agent-slots.js";
import { workloadDefinitions } from "./workloads.js";
const T0 = Date.parse("2026-11-09T12:00:00Z");
@@ -48,6 +50,33 @@ describe("U1 liveness", () => {
expect(result.detail).toMatch(/^no_owner_no_action .* in todo with r1 queued/);
});
+ it("lets a run wait in a declared queue while its agent's slots are full, and still fails one queued with a free slot (w-20261113T052703-s7301 t030)", () => {
+ const queuedThenStarted = [...polls(0, 486 * SEC, at => sample(at, "todo", [run("r1", "queued", 0)])),
+ ...polls(489 * SEC, 600 * SEC, at => sample(at, "in_progress", [run("r1", "running", 486 * SEC, null, { lastEvidenceAt: iso(at) })]))];
+ const declared = { ...BOUNDS, declaredQueuedMs: declaredQueueBoundMs({ runs: 20, queue: 20, turnMinutes: [12, 22] }) };
+ expect(declared.declaredQueuedMs).toBe(27 * 60 * SEC);
+ expect(gradeU1(record(queuedThenStarted), BOUNDS)).toMatchObject({ verdict: "fail", detail: expect.stringMatching(/^no_owner_no_action .* in todo with r1 queued/) });
+ expect(gradeU1(record(queuedThenStarted), declared, () => false).verdict).toBe("pass");
+ expect(gradeU1(record(queuedThenStarted), declared, () => null).verdict).toBe("pass");
+ const free = gradeU1(record(queuedThenStarted), declared, () => true);
+ expect(free).toMatchObject({ verdict: "fail", detail: expect.stringMatching(/r1 queued \d+s \(\d+s with a free slot, declared queue bound 1620s\)/) });
+ const pastDeclared = polls(0, 31 * 60 * SEC, at => sample(at, "todo", [run("r1", "queued", 0)]));
+ expect(gradeU1(record(pastDeclared, { restAt: null, finalStatus: null, timedOut: true }), declared, () => false).verdict).toBe("fail");
+ expect(declaredQueueBoundMs({ runs: 20, queue: 0, turnMinutes: [12, 22] })).toBeUndefined();
+ expect(declaredQueueBoundMs(undefined)).toBeUndefined();
+ });
+
+ it("reads free slots from the agent's own runs and its run cap", () => {
+ const slotFree = slotFreeFrom({ agents: [{ agentId: "a1", maxConcurrentRuns: 2 }, { agentId: "a2", maxConcurrentRuns: null }],
+ runs: [{ runId: "q", agentId: "a1", startedAt: 500, finishedAt: null }, { runId: "x", agentId: "a1", startedAt: 0, finishedAt: 300 },
+ { runId: "y", agentId: "a1", startedAt: 100, finishedAt: null }, { runId: "z", agentId: "a2", startedAt: 0, finishedAt: null }] });
+ expect(slotFree("q", 200)).toBe(false);
+ expect(slotFree("q", 400)).toBe(true);
+ expect(slotFree("z", 10)).toBeNull();
+ expect(slotFree("unknown", 10)).toBeNull();
+ expect(slotFreeFrom(null)("q", 1)).toBeNull();
+ });
+
it("treats a visible Retry or a waiting question as actionable", () => {
const failedRun = run("r1", "failed", 0, 5 * SEC, { errorCode: "process_lost" });
const withRetry = polls(0, 10 * 60 * SEC, at => sample(at, "in_progress", [failedRun], { interruption: retryable }));
diff --git a/tests/restart-chaos/universal.ts b/tests/restart-chaos/universal.ts
index 8306aa081c1..0fdf88ac07f 100644
--- a/tests/restart-chaos/universal.ts
+++ b/tests/restart-chaos/universal.ts
@@ -32,6 +32,14 @@ export interface UniversalBounds {
/** U4: longest a failure or Retry notice may remain after a later run succeeded. */
staleNoticeMs: number;
maxRuns: number;
+ declaredQueuedMs?: number;
+}
+
+export type SlotFree = (runId: string, at: number) => boolean | null;
+
+export function declaredQueueBoundMs(mix: { runs: number; queue: number; turnMinutes: readonly [number, number] } | undefined): number | undefined {
+ if (!mix || mix.queue <= 0) return undefined;
+ return Math.ceil(mix.queue / Math.max(1, mix.runs)) * (mix.turnMinutes[1] + 5) * 60_000;
}
/**
@@ -140,6 +148,7 @@ export function liveOwner(sample: TaskSample, queuedFor: ReadonlyMap {
+export function gradeU1(record: TaskRecord, bounds: UniversalBounds, slotFree: SlotFree = () => null): Omit {
if (!record.samples.some(ready)) return { verdict: "n/a", detail: "no ready sample" };
// Ready-server time each run has spent queued (outages do not count).
+ const totalQueued = new Map();
+ const freeQueued = new Map();
const queuedFor = new Map();
let orphanMs = 0, worstOrphan = 0, orphanFrom: number | null = null, lastAt: number | null = null;
let worstQueued = 0, worstQueuedRun = "";
@@ -172,9 +183,14 @@ export function gradeU1(record: TaskRecord, bounds: UniversalBounds): Omit worstQueued) { worstQueued = waited; worstQueuedRun = run.id; }
+ if (bounds.declaredQueuedMs === undefined || run.status !== "queued") { queuedFor.set(run.id, waited); continue; }
+ const free = (freeQueued.get(run.id) ?? 0) + (slotFree(run.id, sample.at) === true ? step : 0);
+ freeQueued.set(run.id, free);
+ queuedFor.set(run.id, waited > bounds.declaredQueuedMs ? Number.POSITIVE_INFINITY : free);
}
}
timerFor = retryTimer(sample) && sample.interruption?.retryWait ? timerFor + (lastAt !== null ? sample.at - lastAt : 0) : 0;
@@ -189,7 +205,8 @@ export function gradeU1(record: TaskRecord, bounds: UniversalBounds): Omit bounds.orphanMs) {
const queued = (sample.runs ?? []).filter(run => run.status === "queued" || run.status === "scheduled_retry");
return { verdict: "fail", metrics: { ownerlessMs: orphanMs, queuedMs: worstQueued, retryTimerMs: worstTimer }, detail: `no_owner_no_action ${seconds(orphanMs)} in ${sample.issueStatus ?? "unknown"}`
- + (queued.length ? ` with ${queued.map(run => `${run.id} queued ${seconds(queuedFor.get(run.id) ?? 0)}`).join(",")}` : "")
+ + (queued.length ? ` with ${queued.map(run => `${run.id} queued ${seconds(totalQueued.get(run.id) ?? 0)}`
+ + `${freeQueued.has(run.id) ? ` (${seconds(freeQueued.get(run.id)!)} with a free slot, declared queue bound ${seconds(bounds.declaredQueuedMs!)})` : ""}`).join(",")}` : "")
+ (timerFor ? ` after a retry wait of ${seconds(timerFor)}${sample.interruption?.retryWait ? " (accepted Retry waiting on the system)" : ""}` : "")
+ ` (interruption ${sample.interruption?.state ?? "none"})` };
}
@@ -304,11 +321,11 @@ export function gradeU5(record: TaskRecord, bounds: UniversalBounds): Omit } = {}): UniversalGrade {
+ enforced?: ReadonlySet; slotFree?: SlotFree } = {}): UniversalGrade {
const bounds = options.bounds ?? universalBounds();
const enforced = options.enforced ?? ENFORCED_UNIVERSAL;
const results = {
- U1: gradeU1(record, bounds),
+ U1: gradeU1(record, bounds, options.slotFree),
U2: gradeU2(record, bounds),
U3: gradeU3(record, options.db),
U4: gradeU4(record, bounds),
diff --git a/tests/restart-chaos/wake-evidence.test.ts b/tests/restart-chaos/wake-evidence.test.ts
index 2ff7ba29eba..438a0836b55 100644
--- a/tests/restart-chaos/wake-evidence.test.ts
+++ b/tests/restart-chaos/wake-evidence.test.ts
@@ -129,13 +129,17 @@ describe.skipIf(!support.supported)("wake evidence against real PostgreSQL", ()
it("writes one file per task with the wake row, context snapshot, status changes and serving controller", async () => {
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), wakeId = randomUUID(), runId = randomUUID(), bootId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Wake evidence", issuePrefix: companyId.slice(0, 8) });
- await db.insert(agents).values({ id: agentId, companyId, name: "Agent", adapterType: "process", status: "active" });
+ await db.insert(agents).values({ id: agentId, companyId, name: "Agent", adapterType: "process", status: "active",
+ runtimeConfig: { heartbeat: { enabled: false, maxConcurrentRuns: 3 } } });
+ const wideAgentId = randomUUID();
+ await db.insert(agents).values({ id: wideAgentId, companyId, name: "Wide agent", adapterType: "process", status: "active",
+ runtimeConfig: { heartbeat: { enabled: false, maxConcurrentRuns: 100 } } });
await db.insert(issues).values({ id: issueId, companyId, title: "Task" });
await db.insert(controllerInstances).values({ bootId, processSpace: "space-wake", pid: 4242, leaseExpiresAt: new Date(Date.now() + 60_000) });
await db.insert(agentWakeupRequests).values({ id: wakeId, companyId, agentId, source: "on_demand", reason: "retry", triggerDetail: "manual",
requestedByActorType: "user", requestedByActorId: "u1", idempotencyKey: "idem-1", payload: { issueId }, runId });
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, status: "succeeded", invocationSource: "on_demand", wakeupRequestId: wakeId,
- contextSnapshot: { wakeReason: "retry", taskId: issueId }, controllerBootId: bootId });
+ contextSnapshot: { wakeReason: "retry", taskId: issueId }, controllerBootId: bootId, startedAt: new Date(1_800_000_000_000), finishedAt: new Date(1_800_000_060_000) });
await db.insert(activityLog).values({ companyId, actorType: "agent", actorId: agentId, action: "issue.updated", entityType: "issue", entityId: issueId,
details: { status: "done", _previous: { status: "in_progress" } } });
const waveDir = await mkdtemp(path.join(os.tmpdir(), "wake-evidence-"));
@@ -146,6 +150,9 @@ describe.skipIf(!support.supported)("wake evidence against real PostgreSQL", ()
const evidence = JSON.parse(await readFile(path.join(waveDir, "wake-evidence", "t01.json"), "utf8"));
expect(evidence.runs[0]).toMatchObject({ runId, invocationSource: "on_demand", wakeupRequestId: wakeId, contextSnapshot: { wakeReason: "retry" },
owner: { pid: 4242, processSpace: "space-wake" } });
+ const slots = JSON.parse(await readFile(path.join(waveDir, "wake-evidence", "agent-slots.json"), "utf8"));
+ expect(slots.agents).toEqual(expect.arrayContaining([{ agentId, maxConcurrentRuns: 3 }, { agentId: wideAgentId, maxConcurrentRuns: 50 }]));
+ expect(slots.runs).toEqual([{ runId, agentId, startedAt: 1_800_000_000_000, finishedAt: 1_800_000_060_000 }]);
expect(evidence.wakes[0]).toMatchObject({ id: wakeId, reason: "retry", source: "on_demand", triggerDetail: "manual", requestedByActorType: "user",
requestedByActorId: "[REDACTED]", idempotencyKey: "idem-1", runId });
expect(evidence.wakes[0].servingAtCreate.map((server: { pid: number }) => server.pid)).toEqual([4242]);
diff --git a/tests/restart-chaos/wake-evidence.ts b/tests/restart-chaos/wake-evidence.ts
index 7bf4eaf0a51..4cb126718a1 100644
--- a/tests/restart-chaos/wake-evidence.ts
+++ b/tests/restart-chaos/wake-evidence.ts
@@ -4,7 +4,10 @@ import type { ServerProcessWindow } from "./multi-server.js";
import { unshiftJson, unshiftMs, type AgingShift } from "./time-faults.js";
import { ISSUE_STATUSES } from "../../packages/shared/src/constants.js";
import type { CompletionEvidence } from "./staging-mix.js";
-import { createDb, type Db } from "../../packages/db/src/index.js";
+import { laterAnswers, type DeliveryRun, type DeliveryWake } from "./follow-up-delivery.js";
+import { AGENT_SLOTS_FILE, agentSlotsFromRows } from "./agent-slots.js";
+import { agents, createDb, type Db } from "../../packages/db/src/index.js";
+import { maxConcurrentRunsSql } from "../../server/src/services/heartbeat/timer-policy.js";
type Row = Record;
@@ -60,6 +63,7 @@ export interface WakeEvidenceInput {
controllers: readonly ControllerRow[];
windows: readonly ServerProcessWindow[];
issues?: readonly Row[];
+ agents?: readonly Row[];
}
export function wakeEvidenceForIssue(issueId: string, input: WakeEvidenceInput) {
@@ -75,8 +79,7 @@ export function wakeEvidenceForIssue(issueId: string, input: WakeEvidenceInput)
const owner = controllerByBoot.get(String(run.controllerBootId ?? ""));
return { runId: text(run.id), status: text(run.status), invocationSource: text(run.invocationSource), triggerDetail: text(run.triggerDetail),
wakeupRequestId: text(run.wakeupRequestId), createdAt: ms(run.createdAt), startedAt: ms(run.startedAt), finishedAt: ms(run.finishedAt),
- acceptedDisposition: text(run.acceptedDisposition), acceptedAt: ms(run.acceptedAt), contextSnapshot: run.contextSnapshot ?? null,
- owner: owner ? { pid: owner.pid, bootId: owner.bootId, epoch: owner.epoch, processSpace: owner.processSpace } : null,
+ acceptedDisposition: text(run.acceptedDisposition), acceptedAt: ms(run.acceptedAt), contextSnapshot: run.contextSnapshot ?? null, owner: owner ? { pid: owner.pid, bootId: owner.bootId, epoch: owner.epoch, processSpace: owner.processSpace } : null,
servingAtCreate: servingAt(ms(run.createdAt), controllers, input.windows) };
}),
wakes: wakes.map(wake => ({ id: text(wake.id), reason: text(wake.reason), source: text(wake.source), triggerDetail: text(wake.triggerDetail),
@@ -89,10 +92,10 @@ export function wakeEvidenceForIssue(issueId: string, input: WakeEvidenceInput)
};
}
-async function readRows(db: Pick, companyId: string, shifts: readonly AgingShift[]): Promise> {
+async function readRows(db: Pick, companyId: string, shifts: readonly AgingShift[]): Promise> {
const client = db.$client;
const runs = await client.unsafe(
- `select r.id::text as id, r.issue_id::text as "issueId", r.status, r.invocation_source as "invocationSource", r.trigger_detail as "triggerDetail",
+ `select r.id::text as id, r.agent_id::text as "agentId", r.issue_id::text as "issueId", r.status, r.invocation_source as "invocationSource", r.trigger_detail as "triggerDetail",
r.wakeup_request_id::text as "wakeupRequestId", r.context_snapshot as "contextSnapshot", r.controller_boot_id::text as "controllerBootId",
floor(extract(epoch from r.created_at) * 1000) as "createdAt", floor(extract(epoch from r.started_at) * 1000) as "startedAt",
floor(extract(epoch from r.finished_at) * 1000) as "finishedAt", accepted.disposition as "acceptedDisposition",
@@ -117,6 +120,8 @@ async function readRows(db: Pick, companyId: string, shifts: read
floor(extract(epoch from created_at) * 1000) as "createdAt"
from activity_log where company_id::text = $1 and entity_type = 'issue' and action like 'issue.%'
and details ? 'status' order by created_at`, [companyId]) as Row[];
+ const agentRows = (await db.select({ id: agents.id, companyId: agents.companyId, maxConcurrentRuns: maxConcurrentRunsSql() }).from(agents))
+ .filter(agent => agent.companyId === companyId) as Row[];
const controllers = (await client.unsafe(
`select boot_id as "bootId", epoch, process_space as "processSpace", pid, floor(extract(epoch from started_at) * 1000) as "startedAt",
floor(extract(epoch from coalesce(stopped_at, fenced_at)) * 1000) as "endedAt" from controller_instances order by epoch`) as Row[])
@@ -125,6 +130,7 @@ async function readRows(db: Pick, companyId: string, shifts: read
return {
runs: unshiftRows(runs, ["createdAt", "startedAt", "finishedAt", "acceptedAt"], ["contextSnapshot"], shifts),
issues: unshiftRows(issues, ["completedAt"], [], shifts),
+ agents: agentRows,
wakes: unshiftRows(wakes, ["createdAt", "requestedAt", "claimedAt", "finishedAt"], ["payload"], shifts),
statusChanges: unshiftRows(statusChanges, ["createdAt"], [], shifts),
controllers: controllers.map(controller => ({ ...controller, startedAt: unshiftMs(controller.startedAt, shifts),
@@ -132,7 +138,7 @@ async function readRows(db: Pick, companyId: string, shifts: read
};
}
-export async function captureWakeEvidence(input: { db: Pick; companyId: string; waveDir: string;
+export async function captureWakeEvidence(input: { db: Pick; companyId: string; waveDir: string;
observers: ReadonlyArray | null; windows: readonly ServerProcessWindow[];
shifts?: readonly AgingShift[]; redact: (text: string) => string }): Promise {
const rows = await readRows(input.db, input.companyId, input.shifts ?? []);
@@ -150,6 +156,7 @@ export async function captureWakeEvidence(input: { db: Pick; comp
written += 1;
}
await writeFile(path.join(directory, "servers.json"), `${JSON.stringify({ controllers: rows.controllers, harnessWindows: input.windows }, null, 2)}\n`, { mode: 0o600 });
+ await writeFile(path.join(directory, AGENT_SLOTS_FILE), `${JSON.stringify(agentSlotsFromRows(rows.agents ?? [], rows.runs), null, 2)}\n`, { mode: 0o600 });
return written;
}
@@ -158,15 +165,17 @@ export interface EvidenceObserver {
record: { issueId: string | null; stagingMix?: { completion?: CompletionEvidence } };
}
-export function completionEvidence(evidence: ReturnType): CompletionEvidence {
- return { completedAt: evidence.completedAt, runs: evidence.runs.map(run => ({ runId: run.runId, startedAt: run.startedAt,
- acceptedDisposition: run.acceptedDisposition, acceptedAt: run.acceptedAt })) };
+export function completionEvidence(evidence: { completedAt: number | null; wakes?: readonly DeliveryWake[];
+ runs: ReadonlyArray }): CompletionEvidence {
+ const answers = laterAnswers(evidence);
+ return { completedAt: evidence.completedAt, runs: evidence.runs.map(run => ({ runId: run.runId, startedAt: run.startedAt ?? null,
+ acceptedDisposition: run.acceptedDisposition, acceptedAt: run.acceptedAt, answersFollowUp: Boolean(run.runId && answers.has(run.runId)) })) };
}
export function wakeEvidenceOnce(input: { databaseUrl: () => string; companyId: string; waveDir: string;
observers: () => ReadonlyArray; windows: readonly ServerProcessWindow[];
shifts: readonly AgingShift[]; redact: (text: string) => string; note: (kind: string, data: Record) => void;
- open?: (url: string) => Pick }): () => Promise {
+ open?: (url: string) => Pick }): () => Promise {
let done: Promise | null = null;
return () => done ??= (async () => {
let observers: ReturnType | null = null;
diff --git a/tests/restart-chaos/warm-shutdown.ts b/tests/restart-chaos/warm-shutdown.ts
index 3006b68c68c..cc8bc6e087d 100644
--- a/tests/restart-chaos/warm-shutdown.ts
+++ b/tests/restart-chaos/warm-shutdown.ts
@@ -2,6 +2,7 @@ export interface WarmShutdownCleanup {
environmentDeleted: boolean;
environmentDeleteBlocked: string | null;
remainingSandboxes: string[];
+ undeletedSandboxes?: string[];
}
export interface WarmShutdownCheck {
@@ -34,12 +35,13 @@ export function gradeWarmShutdown(input: { cleanup: WarmShutdownCleanup | null;
if (input.cleanup?.environmentDeleteBlocked) problems.push(`environment delete blocked: ${input.cleanup.environmentDeleteBlocked}`);
else if (input.cleanup && !input.cleanup.environmentDeleted) problems.push("environment was not deleted");
if (input.cleanup?.remainingSandboxes.length) problems.push(`sandboxes still running after cleanup: ${input.cleanup.remainingSandboxes.join(", ")}`);
+ if (input.cleanup?.undeletedSandboxes?.length) problems.push(`wave sandboxes left after the harness deleted them: ${input.cleanup.undeletedSandboxes.join(", ")}`);
for (const line of shutdownCloseFailures) problems.push(`warm session did not close: ${line}`);
return {
verdict: problems.length ? "fail" : "pass",
environmentDeleted: input.cleanup?.environmentDeleted ?? null,
environmentDeleteBlocked: input.cleanup?.environmentDeleteBlocked ?? null,
- leakedSandboxes: input.cleanup?.remainingSandboxes ?? [],
+ leakedSandboxes: [...new Set([...input.cleanup?.remainingSandboxes ?? [], ...input.cleanup?.undeletedSandboxes ?? []])],
shutdownCloseFailures,
idleCloseFailures,
problems,
diff --git a/tests/restart-chaos/wave-grade-inputs.ts b/tests/restart-chaos/wave-grade-inputs.ts
new file mode 100644
index 00000000000..a2d7295e557
--- /dev/null
+++ b/tests/restart-chaos/wave-grade-inputs.ts
@@ -0,0 +1,28 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { AGENT_SLOTS_FILE, slotFreeFrom, type AgentSlots } from "./agent-slots.js";
+import { withChildWaits } from "./child-wait.js";
+import type { WakeEvidenceFile } from "./staging-mix-retry.js";
+import type { StagingMixOptions } from "./staging-mix.js";
+import type { TaskRecord } from "./types.js";
+import { declaredQueueBoundMs, universalBounds, type SlotFree, type UniversalBounds } from "./universal.js";
+
+export interface WaveGradeInputs {
+ queue: { bounds: UniversalBounds; slotFree: SlotFree };
+ slotEvidence: boolean;
+ prepare(taskId: string, record: TaskRecord): Promise;
+}
+
+const readJson = async (file: string): Promise => readFile(file, "utf8").then(text => JSON.parse(text) as T).catch(() => null);
+
+export async function loadWaveGradeInputs(waveDir: string, mix: Pick | undefined,
+ env: NodeJS.ProcessEnv = process.env): Promise {
+ const slots = await readJson(path.join(waveDir, "wake-evidence", AGENT_SLOTS_FILE));
+ const declaredQueuedMs = declaredQueueBoundMs(mix);
+ const bounds = { ...universalBounds(env), ...(declaredQueuedMs === undefined ? {} : { declaredQueuedMs }) };
+ return {
+ queue: { bounds, slotFree: slotFreeFrom(slots) },
+ slotEvidence: slots !== null,
+ prepare: async (taskId, record) => withChildWaits(record, await readJson(path.join(waveDir, "wake-evidence", `${taskId}.json`))),
+ };
+}