test(restart-chaos): grade staging-mix hands-off and click Retry once after the settle

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-12 12:43:11 -06:00
1 parent cdefac664b
commit ddb64d3223
9 files changed
+419 -27

No files matched your search

+29 -8
View File
@@ -1338,6 +1338,10 @@ node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona
node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona --daytona-image $IMG --daytona-warm --daytona-cpu 1 \
--storage s3 --ephemeral-server --workload staging-mix --faults K15 --k15-overlap-seconds 480-540 --mix-runs 15 --mix-queue 3 \
--mix-minutes 40 --mix-fault-spacing-minutes 12 --mix-turn-minutes 5-15 --budget-cents <cap> --seed 7102
# The same short form locally (agents on this Mac, so 10 runs). `--ephemeral-server` runs only on Daytona, so it is left out.
node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env local --storage s3 --workload staging-mix --faults K15 \
--k15-overlap-seconds 480-540 --mix-runs 10 --mix-queue 3 --mix-minutes 40 --mix-fault-spacing-minutes 12 --mix-turn-minutes 5-15 \
--budget-cents 5000 --seed 7201
# Full form (about 4.5 hours): 6 swaps 40 minutes apart, 15-45 minute turns, 30 runs and a queue of 6.
RESTART_CHAOS_DAYTONA_VCPU_BUDGET=<org headroom> node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona \
--daytona-image $IMG --daytona-warm --daytona-cpu 1 --storage s3 --ephemeral-server --workload staging-mix --faults K15 \
@@ -1381,13 +1385,29 @@ one settled and at least one task is in flight; it hits every task in flight (`s
tasks do: the wave's clock crosses them. A slot that never fired because every task rested first fails the wave as a
harness error (`staging_mix_fault_slots_missed`). With `--faults K0` the tasks are `K0` and nothing fires.
**Follow-ups and Retries.** About half of the tasks (`--mix-follow-up-share`) get a board comment while a run is
running. The comment asks the agent to add `FOLLOWUP-<nonce>` to the ledger. A failed task's Retry is clicked the
same way. Both are held until the next overlap window, but for at most `--mix-hold-minutes` (default 10); a follow-up
also goes out before the last 15% of the planned turn. A failed post is tried again on the next poll. The
overlap window comes from `setOverlapProbe` in `staging-mix.ts`, which a swap driver registers. Without a probe, the
window is any time a fault is in progress and the server still answers (a K15 overlap). The timeline records
`staging_mix_follow_up` and `staging_mix_retry_in_overlap`.
**Follow-ups.** About half of the tasks (`--mix-follow-up-share`) get a board comment while a run is running. The
comment asks the agent to add `FOLLOWUP-<nonce>` to the ledger. It is held until the next overlap window, but for at
most `--mix-hold-minutes` (default 10), and it goes out before the last 15% of the planned turn. A failed post is
tried again on the next poll with the same `clientRequestId`. The overlap window comes from `setOverlapProbe` in
`staging-mix.ts`, which a swap driver registers. Without a probe, the window is any time a fault is in progress and the
server still answers (a K15 overlap). The timeline records `staging_mix_follow_up`.
**Retries: hands-off first.** By default the harness never clicks Retry during the wave or the settle, so the product's
own continuation is what gets measured. A task that shows a Retry banner is kept under watch, not rested. Every banner
is recorded per task: when it first showed, how long it stayed, and the error code behind it, even when the task then
finished by itself. A banner is a defect, because the product goal is that it never appears. After the settle
(`--mix-settle-minutes`, default 15, after the last fault slot ends), each task that is still not done and still
offers Retry gets one click (`staging_mix_retry_after_settle`), and the harness follows it to rest. When the release-swap
cell drives the wave, its own end-of-wave pass makes that one click instead. Both paths use
`shouldClickRetryAfterSettle` in `staging-mix-retry.ts`: never on a done task, a task waiting on a scheduled retry, or a
task that was already clicked. So no task gets two clicks. `--mix-retry-during-wave` restores the old behaviour: Retries
clicked during the wave, held until an overlap window as follow-ups are (`staging_mix_retry_in_overlap`).
**Results.** `staging-mix-results.json` and the first lines of `SUMMARY.md` give three numbers, in this order:
`completed hands-off / started`, `Retry banner shown: N tasks` and `Retry worked / Retry clicked`. A Retry counts as
worked when the task is done at the end. The banner list follows, then one read of
`GET /api/companies/:id/assigned-work-invariant` at the end of the wave: `violationCount`, `violations` by reason,
`agentsWithStalledSlots` and `openTasks`.
**Grade.** The usual R1–R6, U1–U5 and T rules apply. A needed Retry still fails R3. The T-oracle checks the result,
not the status:
@@ -1397,7 +1417,8 @@ not the status:
| `issue_done` | the task is not `done` |
| `ledger_rounds_verified` | a round is missing, ran under 80 s, or its hash differs from the harness's own computation (POSIX `cksum`, chained per tick) |
| `rounds_once` | a round line appears twice (a tool ran again) |
| `done_comment_once` | the `DONE-<nonce>` comment is missing or was posted twice |
| `done_comment_once` | the comment whose whole body is `DONE-<nonce>` is missing or was posted twice; a summary that quotes the marker does not count |
| `completion_once` | more than one run posted a comment that mentions `DONE-<nonce>` (a duplicate completion) |
| `follow_up_posted` | a planned follow-up was never posted (the detail lists the post errors) |
| `follow_up_served` | a follow-up was posted during a run and the ledger lacks its line |
| `subtasks_once`, `subtask_<k>_done`, `parent_read_subtask_<k>` | a parent created more or fewer than two subtasks, a subtask's ledger fails, or the parent did not report a subtask's last hash |
+6 -8
View File
@@ -49,6 +49,7 @@ import { chaosLifecycleOverrides, seedChaosLab } from "./seed.js";
import { ChaosSupervisor } from "./supervisor.js";
import { planStagingMixWave } from "./staging-mix.js";
import { StagingMixRun, stagingMixSeedOptions } from "./staging-mix-run.js";
import { writeStagingMixResults } from "./staging-mix-retry.js";
import { TimeShiftedApi } from "./time-faults.js";
import type { RestartStep } from "./repeated-restarts.js";
import { isBrokenStdoutError, markTimelineEchoBroken, Timeline } from "./timeline.js";
@@ -734,13 +735,8 @@ async function main(): Promise<number> {
sandboxWatchStopped = true;
await mix?.stop();
await Promise.all(faultWork);
const mixShortfall = mix && !interrupted ? mix.shortfall(launched.length) : [];
if (mixShortfall.length) {
timeline.add("staging_mix_shortfall", { problems: mixShortfall, unfired: mix!.unfiredSlots() });
harnessError ??= new Error(`Staging mix tested less than planned: ${mixShortfall.join("; ")}`);
}
if (mix) console.info(`Staging mix: ${launched.length} tasks started; ${mix.unlaunched} planned tasks never launched (excluded from tasks started); `
+ `fault slots fired ${mix.slotCoverage.map(entry => `${Math.round(entry.atMs / 60_000)}m:${entry.kind}×${entry.exposed} tasks`).join(", ") || "none"} of ${mix.slots.length}`);
harnessError ??= mix && !interrupted ? mix.report(launched.length) ?? undefined : undefined;
if (mix && !interrupted && !releaseSwap) await mix.retryAfterSettle(launched, api, () => interrupted);
if (storageTrigger) {
const shortfall = storageTriggerShortfall(storageTrigger.finish(faults));
storageSection = storageTriggerMarkdown(storageTrigger.receipts);
@@ -840,6 +836,8 @@ async function main(): Promise<number> {
rules: Object.fromEntries(Object.entries(grade.rules).map(([id, rule]) => [id, rule.verdict])), T: grade.T.verdict,
universal: Object.fromEntries(Object.entries(grade.universal ?? {}).map(([id, rule]) => [id, `${rule.verdict}${rule.enforced ? "" : " (warning)"}`])) });
}
if (mix) await writeStagingMixResults({ waveDir, api, companyId: lab.companyId, tasks: launched, passed: new Map(lines.map(line => [line.taskId, line.pass])), timeline: timeline.entries })
.catch(error => { harnessError ??= new Error(`Staging-mix results failed: ${String(error)}`); });
// MS1–MS3: the cross-server invariant over the whole wave (multi-server.ts).
if (faults.some(receipt => receipt.multiServer)) {
const crossServer = await readCrossServerEvidence(evidenceDb, lab.companyId)
@@ -987,7 +985,7 @@ async function main(): Promise<number> {
if (lines.length) {
await appendLedger(resultsRoot, lines);
const summary = await writeSummary(resultsRoot, plan.waveId);
await writeFile(path.join(waveDir, "SUMMARY.md"), [await readFile(path.join(waveDir, "release-swap-summary.md"), "utf8").catch(() => null), summary, warmShutdownSection, speedSection, storageSection].filter(Boolean).join("\n"), { mode: 0o600 });
await writeFile(path.join(waveDir, "SUMMARY.md"), [...await Promise.all(["staging-mix-summary.md", "release-swap-summary.md"].map(file => readFile(path.join(waveDir, file), "utf8").catch(() => null))), summary, warmShutdownSection, speedSection, storageSection].filter(Boolean).join("\n"), { mode: 0o600 });
console.info(`\n${summary}`);
}
}
+13 -4
View File
@@ -20,7 +20,7 @@ import { awaitingQuestionAnswer, collectQuestionArtifacts, questionStep } from "
import { isSessionFault, rotateAgentSession } from "./session-faults.js";
import { awaitingRewake, collectRewakeArtifacts, isRewakeWorkload, rewakeStep } from "./rewake.js";
import { withLongTurn } from "./long-turn.js";
import { holdsRetry, stagingMixArtifacts, stagingMixDefinition, stagingMixStep } from "./staging-mix.js";
import { collectStagingMixArtifacts, holdsRetry, stagingMixDefinition, stagingMixStep } from "./staging-mix.js";
import { captureRemoteWorkspaceFiles, readRemoteWorkspaceFile } from "./remote-workspace-files.js";
import { taskFaultSchedule } from "./scheduler.js";
import { g1ChildTitle, g1Decision, workloadDefinitions, type WorkloadAgents } from "./workloads.js";
@@ -686,8 +686,9 @@ export class TaskObserver {
}
if (!await this.clearDiskFullAtRest(now)) { this.restCandidateSince = null; return; }
if (status !== "done" && (!this.record.retry || faultSinceRetry) && sample.interruption?.canRetry) {
if (this.plan.stagingMix && holdsRetry(this.record, this.plan.stagingMix.holdMs, now)) { this.restCandidateSince = null; return; }
await this.boardRetry(sample, now);
if (this.plan.stagingMix?.retryDuringWave && holdsRetry(this.record, this.plan.stagingMix.holdMs, now)) { this.restCandidateSince = null; return; }
if (!this.plan.stagingMix || this.plan.stagingMix.retryDuringWave) { await this.boardRetry(sample, now); return; }
this.restCandidateSince = null;
return;
}
this.record.restAt = this.restCandidateSince ?? now;
@@ -697,6 +698,14 @@ export class TaskObserver {
canRetry: sample.interruption?.canRetry ?? false, retried: Boolean(this.record.retry) });
}
async retryAfterSettle(sample: TaskSample, now: number): Promise<void> {
this.done = false;
this.record.restAt = null;
this.record.timedOut = false;
await this.boardRetry(sample, now);
this.record.deadlineAt = now + RETRY_BUDGET_MS;
}
private async boardRetry(sample: TaskSample, now: number): Promise<void> {
if (this.record.retry) (this.record.priorRetries ??= []).push(this.record.retry);
this.record.retry = await this.retry(sample, now);
@@ -1092,7 +1101,7 @@ export class TaskObserver {
status: String(run.status), errorCode: run.errorCode ?? null, createdAt: run.createdAt ?? null, finishedAt: run.finishedAt ?? null }))).catch(() => []),
...(this.journey ? { journey: await this.journey.collect(issueId).finally(() => this.journey!.close()) } : {}),
...(this.plan.workload === "H2" ? { handoff: await this.collectHandoff(issueId, issue) } : {}),
...(this.plan.stagingMix ? { stagingMix: stagingMixArtifacts(this.record) } : {}),
...(this.plan.stagingMix ? { stagingMix: await collectStagingMixArtifacts(this.api, issueId, this.record) } : {}),
};
}
+2 -1
View File
@@ -9,6 +9,7 @@ import {
gradeReleaseSwapTask, overlappingRuns, releaseSwapMarkdown, summarizeGrades, swapMetrics,
type DuplicateFacts, type GradeTaskInput, type RetryClick, type SwapRunFact, type TaskGrade,
} from "./release-swap-grade.js";
import { issueView, shouldClickRetryAfterSettle } from "./staging-mix-retry.js";
import type { TimelineEntry } from "./timeline.js";
import type { Db } from "../../packages/db/src/index.js";
@@ -208,7 +209,7 @@ export async function finishReleaseSwap(input: ReleaseSwapFinishInput): Promise<
for (const issue of issues) {
const id = String(issue.id);
const current = views.get(id)!;
if (current.status === "done" || current.interruption?.canRetry !== true || current.interruption?.retryWait) continue;
if (!shouldClickRetryAfterSettle(issueView(current), clicks.get(id)?.length ?? 0)) continue;
const click = await clickRetry(input, id, text(issue.identifier) ?? id, text(current.interruption?.observedRunId));
clicks.set(id, [...(clicks.get(id) ?? []), click]);
retried.push(id);
@@ -0,0 +1,130 @@
import { describe, expect, it } from "vitest";
import { parseCommand } from "./config.js";
import { doneMarker, expectedRoundHash, gradeStagingMix, planStagingMixWave, ROUND_SECONDS, type StagingMixTask } from "./staging-mix.js";
import {
readAssignedWorkInvariant, retryAfterSettleStage, shouldClickRetryAfterSettle, stagingMixHeadline, summarizeStagingMix, type RetryStageObserver,
} from "./staging-mix-retry.js";
import type { OracleArtifacts, TaskPlan, TaskRecord, TaskSample } from "./types.js";
const wave = (argv: string[]) => {
const command = parseCommand(argv);
if (command.mode !== "wave") throw new Error("expected a wave");
return command.options;
};
const task = (overrides: Partial<StagingMixTask> = {}): StagingMixTask => ({
turnMinutes: 5, rounds: 4, subtasks: 0, followUpAfterMs: null, arriveAtMs: 0, holdMs: 600_000, ...overrides,
});
const ledger = (nonce: string, rounds: number) => Array.from({ length: rounds }, (_, index) =>
`ROUND ${index + 1} ${1_800_000_000 + index * 100} ${1_800_000_000 + index * 100 + ROUND_SECONDS} ${expectedRoundHash(nonce, index + 1)}`).join("\n");
describe("staging-mix Retry policy", () => {
it("is hands-off by default, and keeps the overlap Retry only behind --mix-retry-during-wave", () => {
const base = ["--workload", "staging-mix", "--faults", "K0", "--mix-turn-minutes", "5-15", "--mix-minutes", "40"];
const handsOff = wave(base);
expect(handsOff.stagingMix).toMatchObject({ retryDuringWave: false, settleMinutes: 15 });
expect(planStagingMixWave("w", handsOff).tasks.every(entry => entry.stagingMix!.retryDuringWave === undefined)).toBe(true);
const old = wave([...base, "--mix-retry-during-wave", "--mix-settle-minutes", "5"]);
expect(old.stagingMix).toMatchObject({ retryDuringWave: true, settleMinutes: 5 });
expect(planStagingMixWave("w", old).tasks.every(entry => entry.stagingMix!.retryDuringWave === true)).toBe(true);
});
it("clicks once after the settle, only on a task still offering Retry that nobody clicked", () => {
expect(shouldClickRetryAfterSettle({ status: "blocked", canRetry: true, retryWait: false }, 0)).toBe(true);
expect(shouldClickRetryAfterSettle({ status: "blocked", canRetry: true, retryWait: false }, 1)).toBe(false);
expect(shouldClickRetryAfterSettle({ status: "done", canRetry: true, retryWait: false }, 0)).toBe(false);
expect(shouldClickRetryAfterSettle({ status: "blocked", canRetry: true, retryWait: true }, 0)).toBe(false);
expect(shouldClickRetryAfterSettle({ status: "in_progress", canRetry: false, retryWait: false }, 0)).toBe(false);
});
it("waits out the settle hands-off, then clicks once per stuck task and follows it to rest", async () => {
let clock = 0;
const issues: Record<string, Record<string, unknown>> = {
i1: { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r1" } },
i2: { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r2" } },
i3: { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r3" } },
};
const observer = (taskId: string, issueId: string, retry: TaskRecord["retry"] = null): RetryStageObserver & { clicks: number } => {
const self = {
plan: { taskId, stagingMix: task() } as TaskPlan, record: { issueId, retry } as TaskRecord, done: true, clicks: 0,
async poll() { if (issues[issueId]!.status === "done") self.done = true; },
async retryAfterSettle(sample: TaskSample, at: number) {
self.done = false; self.clicks += 1;
expect(sample.interruption?.observedRunId).toBe(`r${issueId.slice(1)}`);
self.record.retry = { at, accepted: true } as TaskRecord["retry"];
issues[issueId] = { status: "done" };
},
};
return self;
};
const healsItself = observer("t1", "i1"), stuck = observer("t2", "i2"), alreadyClicked = observer("t3", "i3", { at: 1, accepted: true } as TaskRecord["retry"]);
const notes: string[] = [];
const result = await retryAfterSettleStage({ observers: [healsItself, stuck, alreadyClicked], api: { get: async <T>(path: string) => issues[path.split("/").pop()!] as T },
settleUntil: 120_000, settleMs: 60_000, cancelled: () => false, note: kind => notes.push(kind), pollMs: 60_000, now: () => clock,
sleep: async ms => { clock += ms; if (clock >= 60_000) issues.i1 = { status: "done" }; } });
expect(clock).toBeGreaterThanOrEqual(120_000);
expect(result.clicked).toEqual(["t2"]);
expect([healsItself.clicks, stuck.clicks, alreadyClicked.clicks]).toEqual([0, 1, 0]);
expect(stuck.done).toBe(true);
expect(healsItself.record.stagingMix!.retryBanner).toMatchObject({ firstShownAt: 0, shownMs: 60_000 });
expect(stuck.record.stagingMix!.retryAfterSettle).toMatchObject({ accepted: true, issueStatusAtClick: "blocked" });
expect(notes).toEqual(["staging_mix_settling", "staging_mix_retry_after_settle"]);
});
it("reports the three numbers first, counting the swap grader's own clicks and banners that healed by themselves", () => {
const record = (overrides: Partial<TaskRecord> = {}) => ({ issueId: null, retry: null, ...overrides }) as TaskRecord;
const banner = { firstShownAt: 0, lastShownAt: 30_000, shownMs: 30_000, visibleSince: null, errorCode: "controller_lease_lost" };
const result = summarizeStagingMix([
{ taskId: "t1", issueId: "i1", record: record(), pass: true, finalStatus: "done" },
{ taskId: "t2", issueId: "i2", record: record({ stagingMix: { retryBanner: banner } }), pass: true, finalStatus: "done" },
{ taskId: "t3", issueId: "i3", record: record({ retry: { at: 1, accepted: true } as TaskRecord["retry"], stagingMix: { retryBanner: banner } }), pass: false, finalStatus: "done" },
{ taskId: "t4", issueId: "i4", record: record({ stagingMix: { retryBanner: banner } }), pass: false, finalStatus: "blocked" },
{ taskId: "t5", issueId: "i5", record: record(), pass: false, finalStatus: "in_progress" },
], new Map([["i4", true]]), { violationCount: 2, violations: { nothing_pending: 2 }, agentsWithStalledSlots: 1, openTasks: 3 }, 60_000);
expect(stagingMixHeadline(result)).toEqual([
"completed hands-off / started: 2 / 5",
"Retry banner shown: 3 tasks",
"Retry worked / Retry clicked: 1 / 2",
]);
expect(result.bannerShown.find(entry => entry.taskId === "t2")).toMatchObject({ completedAnyway: true, errorCode: "controller_lease_lost", shownMs: 30_000 });
expect(result.invariant.violationCount).toBe(2);
});
it("reads the assigned-work invariant once as counts, and records a failed read", async () => {
const report = { checked: 4, openTasks: 3, violations: { nothing_pending: 1 }, violationCount: 1, agentsWithStalledSlots: [{ agentId: "a" }] };
expect(await readAssignedWorkInvariant({ get: async <T>() => report as T }, "c1"))
.toEqual({ violationCount: 1, violations: { nothing_pending: 1 }, agentsWithStalledSlots: 1, openTasks: 3 });
expect(await readAssignedWorkInvariant({ get: async () => { throw new Error("HTTP 404"); } }, "c1")).toMatchObject({ violationCount: null, error: "Error: HTTP 404" });
});
});
describe("staging-mix completion comments", () => {
const nonce = "b3d70e77";
const base = (comments: Array<{ body: string; runId: string | null }>): OracleArtifacts => ({ issueStatus: "done", documents: { ledger: ledger(nonce, 4) },
agentComments: comments.map(comment => comment.body), files: {}, commands: {}, workspacePath: null,
stagingMix: { followUp: null, followUpErrors: [], retries: [], comments } });
const check = (artifacts: OracleArtifacts, name: string) => gradeStagingMix(nonce, task(), artifacts).checks.find(entry => entry.name === name)!;
it("counts only the DONE comment itself, not a closing summary that quotes it", () => {
const comments = [
{ body: `CHA-1 complete. Saved the ledger and posted \`${doneMarker(nonce)}\`.`, runId: "run-b" },
{ body: "Prepared CHA-1 ledger for this response.", runId: "run-b" },
{ body: doneMarker(nonce), runId: "run-b" },
];
expect(check(base(comments), "done_comment_once")).toMatchObject({ ok: true, detail: "comments=1" });
expect(check(base(comments), "completion_once").ok).toBe(true);
expect(gradeStagingMix(nonce, task(), base(comments)).verdict).toBe("pass");
expect(check(base([...comments, { body: ` \`${doneMarker(nonce)}\`. `, runId: "run-c" }]), "done_comment_once")).toMatchObject({ ok: false, detail: "comments=2" });
});
it("flags a completion summary repeated by a different run as a duplicate completion", () => {
const comments = [
{ body: `CHA-1 complete. Posted exactly one comment ${doneMarker(nonce)} and marked the task done.`, runId: "413ddcf2-ffce" },
{ body: `CHA-1 complete. Posted \`${doneMarker(nonce)}\`.`, runId: "13de31be-fb13" },
{ body: doneMarker(nonce), runId: "13de31be-fb13" },
];
expect(check(base(comments), "done_comment_once").ok).toBe(true);
expect(check(base(comments), "completion_once")).toMatchObject({ ok: false, detail: "duplicate completion: runs 413ddcf2, 13de31be each reported completion" });
});
});
+152
View File
@@ -0,0 +1,152 @@
import { writeFile } from "node:fs/promises";
import path from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import type { ChaosApi } from "./api.js";
import { bannerShownMs, noteRetryBanner } from "./staging-mix.js";
import type { InterruptionSample, TaskPlan, TaskRecord, TaskSample } from "./types.js";
type Json = Record<string, any>;
export interface IssueView { status: string; canRetry: boolean; retryWait: boolean; observedRunId: string | null; interruption: InterruptionSample | null }
export function shouldClickRetryAfterSettle(view: Pick<IssueView, "status" | "canRetry" | "retryWait">, priorClicks: number): boolean {
return view.status !== "done" && view.canRetry && !view.retryWait && priorClicks === 0;
}
export function issueView(issue: Json | null): IssueView {
const interruption = issue?.interruption ?? null;
return {
status: String(issue?.status ?? "unreadable"),
canRetry: interruption?.canRetry === true,
retryWait: Boolean(interruption?.retryWait),
observedRunId: typeof interruption?.observedRunId === "string" ? interruption.observedRunId : null,
interruption: interruption ? { state: String(interruption.state), canRetry: interruption.canRetry === true, retryWait: Boolean(interruption.retryWait),
workspaceFailure: null, observedRunId: typeof interruption.observedRunId === "string" ? interruption.observedRunId : null,
userAction: typeof interruption.userAction === "string" ? interruption.userAction : null } : null,
};
}
export interface RetryStageObserver {
plan: TaskPlan;
record: TaskRecord;
done: boolean;
poll(now?: number): Promise<void>;
retryAfterSettle(sample: TaskSample, now: number): Promise<void>;
}
export interface RetryStageInput {
observers: readonly RetryStageObserver[];
api: Pick<ChaosApi, "get">;
settleUntil: number;
settleMs: number;
cancelled: () => boolean;
note: (kind: string, data: Record<string, unknown>) => void;
pollMs?: number;
now?: () => number;
sleep?: (ms: number) => Promise<void>;
}
export async function retryAfterSettleStage(input: RetryStageInput): Promise<{ clicked: string[] }> {
const now = input.now ?? Date.now;
const sleep = input.sleep ?? (ms => delay(ms));
const pollMs = input.pollMs ?? 60_000;
const tracked = input.observers.filter(observer => observer.record.issueId && observer.plan.stagingMix);
const views = new Map<RetryStageObserver, IssueView>();
const read = async (observers: readonly RetryStageObserver[]) => {
for (const observer of observers) {
const issue = await input.api.get<Json>(`/api/issues/${observer.record.issueId}`, 10_000).catch(() => null);
const view = issueView(issue);
views.set(observer, view);
noteRetryBanner(observer.record.stagingMix ??= {}, view.status !== "done" && view.canRetry, now(), null);
}
};
input.note("staging_mix_settling", { settleUntil: new Date(input.settleUntil).toISOString(), tasks: tracked.length });
for (;;) {
await read(tracked.filter(observer => views.get(observer)?.status !== "done"));
if (input.cancelled() || now() >= input.settleUntil || tracked.every(observer => views.get(observer)?.status === "done")) break;
await sleep(Math.min(pollMs, Math.max(1, input.settleUntil - now())));
}
const clicked: RetryStageObserver[] = [];
for (const observer of tracked) {
const view = views.get(observer);
if (!view || input.cancelled() || !shouldClickRetryAfterSettle(view, observer.record.retry ? 1 : 0)) continue;
const at = now();
await observer.retryAfterSettle({ at, reachable: true, serverReady: true, issueStatus: view.status, interruption: view.interruption }, at);
(observer.record.stagingMix ??= {}).retryAfterSettle = { at, accepted: observer.record.retry?.accepted ?? null, issueStatusAtClick: view.status };
input.note("staging_mix_retry_after_settle", { taskId: observer.plan.taskId, accepted: observer.record.retry?.accepted ?? null });
clicked.push(observer);
}
const until = now() + input.settleMs;
while (clicked.some(observer => !observer.done) && !input.cancelled() && now() < until) {
for (const observer of clicked) if (!observer.done) await observer.poll(now());
await sleep(Math.min(pollMs, 15_000));
}
return { clicked: clicked.map(observer => observer.plan.taskId) };
}
export interface ResultTask { taskId: string; issueId: string | null; record: TaskRecord; pass: boolean; finalStatus: string | null }
export interface StagingMixResult {
started: number;
completedHandsOff: number;
bannerShown: Array<{ taskId: string; firstShownAt: string; shownMs: number; errorCode: string | null; completedAnyway: boolean }>;
retryClicked: number;
retryWorked: number;
invariant: { violationCount: number | null; violations: Record<string, number> | null; agentsWithStalledSlots: number | null; openTasks: number | null; error?: string };
}
export function summarizeStagingMix(tasks: readonly ResultTask[], extraClicks: ReadonlyMap<string, boolean | null>, invariant: StagingMixResult["invariant"], now = Date.now()): StagingMixResult {
const clicked = (task: ResultTask) => Boolean(task.record.retry || task.record.priorRetries?.length || (task.issueId && extraClicks.has(task.issueId)));
const clickedTasks = tasks.filter(clicked);
return {
started: tasks.length,
completedHandsOff: tasks.filter(task => task.pass && !clicked(task)).length,
bannerShown: tasks.flatMap(task => task.record.stagingMix?.retryBanner ? [{ taskId: task.taskId, firstShownAt: new Date(task.record.stagingMix.retryBanner.firstShownAt).toISOString(),
shownMs: bannerShownMs(task.record.stagingMix.retryBanner, now), errorCode: task.record.stagingMix.retryBanner.errorCode,
completedAnyway: task.finalStatus === "done" && !clicked(task) }] : []),
retryClicked: clickedTasks.length,
retryWorked: clickedTasks.filter(task => task.finalStatus === "done").length,
invariant,
};
}
export function stagingMixHeadline(result: StagingMixResult): string[] {
return [
`completed hands-off / started: ${result.completedHandsOff} / ${result.started}`,
`Retry banner shown: ${result.bannerShown.length} tasks`,
`Retry worked / Retry clicked: ${result.retryWorked} / ${result.retryClicked}`,
];
}
export async function readAssignedWorkInvariant(api: Pick<ChaosApi, "get">, companyId: string): Promise<StagingMixResult["invariant"]> {
try {
const report = await api.get<Json>(`/api/companies/${companyId}/assigned-work-invariant`, 30_000);
return { violationCount: typeof report.violationCount === "number" ? report.violationCount : null, violations: report.violations ?? null,
agentsWithStalledSlots: Array.isArray(report.agentsWithStalledSlots) ? report.agentsWithStalledSlots.length : null,
openTasks: typeof report.openTasks === "number" ? report.openTasks : null };
} catch (error) {
return { violationCount: null, violations: null, agentsWithStalledSlots: null, openTasks: null, error: String(error) };
}
}
export async function writeStagingMixResults(input: { waveDir: string; api: Pick<ChaosApi, "get">; companyId: string;
tasks: ReadonlyArray<{ plan: TaskPlan; record: TaskRecord }>; passed: ReadonlyMap<string, boolean>;
timeline: ReadonlyArray<{ kind: string; [key: string]: unknown }> }): Promise<StagingMixResult> {
const extraClicks = new Map<string, boolean | null>(input.timeline.filter(entry => entry.kind === "release_swap_retry_clicked" && typeof entry.issueId === "string")
.map(entry => [String(entry.issueId), entry.accepted === true ? true : entry.accepted === false ? false : null]));
const tasks: ResultTask[] = [];
for (const task of input.tasks) {
const issue = task.record.issueId ? await input.api.get<Json>(`/api/issues/${task.record.issueId}`, 10_000).catch(() => null) : null;
tasks.push({ taskId: task.plan.taskId, issueId: task.record.issueId, record: task.record, pass: input.passed.get(task.plan.taskId) === true,
finalStatus: issue?.status ?? task.record.finalStatus });
}
const result = summarizeStagingMix(tasks, extraClicks, await readAssignedWorkInvariant(input.api, input.companyId));
await writeFile(path.join(input.waveDir, "staging-mix-results.json"), `${JSON.stringify({ schema: "paperclip.restart-chaos.staging-mix/v1",
headline: stagingMixHeadline(result), ...result }, null, 2)}\n`, { mode: 0o600 });
const invariant = result.invariant.violationCount === null ? `unreadable${result.invariant.error ? ` (${result.invariant.error})` : ""}`
: `${result.invariant.violationCount} violations, ${result.invariant.agentsWithStalledSlots} agents with stalled slots, ${result.invariant.openTasks} open tasks`;
await writeFile(path.join(input.waveDir, "staging-mix-summary.md"), [...stagingMixHeadline(result), `Assigned-work invariant: ${invariant}`, "",
...result.bannerShown.map(entry => `- ${entry.taskId}: Retry banner from ${entry.firstShownAt} for ${Math.round(entry.shownMs / 1000)} s`
+ `${entry.errorCode ? ` (${entry.errorCode})` : ""}${entry.completedAnyway ? ", completed without a click" : ""}`), ""].join("\n"), { mode: 0o600 });
return result;
}
+20
View File
@@ -3,7 +3,9 @@ import { readFileSync } from "node:fs";
import { writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import type { ChaosApi } from "./api.js";
import { agentRunCaps, type StagingMixOptions, type StagingMixSlot } from "./staging-mix.js";
import { retryAfterSettleStage, type RetryStageObserver } from "./staging-mix-retry.js";
import type { AdapterId, TaskPlan, WaveOptions } from "./types.js";
export const RESOURCE_SAMPLE_MS = 30_000;
@@ -128,6 +130,24 @@ export class StagingMixRun {
return problems;
}
report(started: number): Error | null {
const problems = this.shortfall(started);
if (problems.length) this.note("staging_mix_shortfall", { problems, unfired: this.unfiredSlots() });
console.info(`Staging mix: ${started} tasks started; ${this.unlaunched} planned tasks never launched (excluded from tasks started); `
+ `fault slots fired ${this.slotCoverage.map(entry => `${Math.round(entry.atMs / 60_000)}m:${entry.kind}×${entry.exposed} tasks`).join(", ") || "none"} of ${this.slots.length}`);
return problems.length ? new Error(`Staging mix tested less than planned: ${problems.join("; ")}`) : null;
}
settleUntil(now = Date.now()): number {
const last = this.slots.at(-1);
return Math.max(now, last ? this.startedAt + last.atMs + last.holdMs : now) + this.mix.settleMinutes * 60_000;
}
async retryAfterSettle(observers: readonly RetryStageObserver[], api: Pick<ChaosApi, "get">, cancelled: () => boolean): Promise<void> {
if (this.mix.retryDuringWave) return;
await retryAfterSettleStage({ observers, api, settleUntil: this.settleUntil(), settleMs: this.mix.settleMinutes * 60_000, cancelled, note: this.note });
}
mayLaunch(now: number, next: TaskPlan): boolean {
if (this.ended || !next.stagingMix || now < this.startedAt + next.stagingMix.arriveAtMs) return false;
const latest = this.samples[this.samples.length - 1];
+12 -1
View File
@@ -264,6 +264,7 @@ describe("staging-mix board step", () => {
const failed: TaskSample = { at: 0, reachable: true, serverReady: true, issueStatus: "blocked",
interruption: { state: "failed", canRetry: true, retryWait: false, workspaceFailure: null, observedRunId: "r1", userAction: null },
runs: [{ ...running.runs![0]!, status: "failed" }] };
const stepIn = (rec: TaskRecord, sample: TaskSample, now: number, faultInProgress: boolean, plan: StagingMixTask) => step(rec, sample, now, faultInProgress, plan);
const step = async (rec: TaskRecord, sample: TaskSample, now: number, faultInProgress: boolean, plan = task({ followUpAfterMs: 60_000 }),
reply: () => unknown = () => ({ id: "c1" })) => {
const posted: string[] = [], retried: number[] = [], notes: string[] = [];
@@ -309,8 +310,18 @@ describe("staging-mix board step", () => {
expect((await step(rec, running, Math.round(4 * ROUND_SECONDS * 1_000 * 0.85), false, plan)).posted.length).toBe(1);
});
it("waits for an overlap to click a Retry, and holds the observer's own click until the hold runs out", async () => {
it("never clicks Retry during the wave by default, but records the banner", async () => {
const rec = record();
expect((await step(rec, failed, 1_000, true)).retried).toEqual([]);
expect((await step(rec, failed, 9_000, true)).retried).toEqual([]);
expect(rec.stagingMix!.retryBanner).toMatchObject({ firstShownAt: 1_000, lastShownAt: 9_000, visibleSince: 1_000 });
await step(rec, running, 20_000, false);
expect(rec.stagingMix!.retryBanner).toMatchObject({ shownMs: 19_000, visibleSince: null });
});
it("with --mix-retry-during-wave, waits for an overlap to click a Retry, and holds the observer's own click until the hold runs out", async () => {
const rec = record();
const step = (r: TaskRecord, sample: TaskSample, now: number, overlap: boolean) => stepIn(r, sample, now, overlap, task({ followUpAfterMs: 60_000, retryDuringWave: true }));
expect((await step(rec, failed, 1_000, false)).retried).toEqual([]);
expect(holdsRetry(rec, 600_000, 2_000)).toBe(true);
expect(holdsRetry(rec, 600_000, 1_000 + 600_000)).toBe(false);
+55 -5
View File
@@ -30,6 +30,8 @@ export interface StagingMixOptions {
minFreeMb: number;
arrivalCutoffMinutes: number;
arrivalEndMs: number;
retryDuringWave: boolean;
settleMinutes: number;
tasks?: number;
}
@@ -42,6 +44,7 @@ export interface StagingMixTask {
followUpAfterMs: number | null;
arriveAtMs: number;
holdMs: number;
retryDuringWave?: boolean;
}
export interface StagingMixRecord {
@@ -49,12 +52,17 @@ export interface StagingMixRecord {
followUp?: { at: number; commentId: string | null; duringRun: boolean; inOverlap: boolean; error: string | null };
retrySeenAt?: number | null;
retries?: Array<{ at: number; inOverlap: boolean }>;
retryBanner?: RetryBanner;
retryAfterSettle?: { at: number; accepted: boolean | null; issueStatusAtClick: string | null };
}
export interface RetryBanner { firstShownAt: number; lastShownAt: number; shownMs: number; visibleSince: number | null; errorCode: string | null }
export interface StagingMixArtifacts {
followUp: StagingMixRecord["followUp"] | null;
followUpErrors: string[];
retries: NonNullable<StagingMixRecord["retries"]>;
comments?: Array<{ body: string; runId: string | null }>;
}
export const STAGING_MIX_ARG_OPTIONS = {
@@ -71,6 +79,8 @@ export const STAGING_MIX_ARG_OPTIONS = {
"mix-run-cap": { type: "string" },
"mix-tasks": { type: "string" },
"mix-arrival-cutoff-minutes": { type: "string" },
"mix-retry-during-wave": { type: "boolean" },
"mix-settle-minutes": { type: "string" },
} as const;
type MixArgs = Partial<Record<keyof typeof STAGING_MIX_ARG_OPTIONS, string | boolean | undefined>> & Record<string, unknown>;
@@ -141,6 +151,8 @@ export function applyStagingMix(options: WaveOptions, values: MixArgs, environme
minFreeMb: whole(environment.RESTART_CHAOS_MIX_MIN_FREE_MB, "RESTART_CHAOS_MIX_MIN_FREE_MB", DEFAULT_MIN_FREE_MB, 0, 1_048_576),
arrivalCutoffMinutes,
arrivalEndMs: arrivalEndMs({ minutes, faultSpacingMinutes, arrivalCutoffMinutes }, options),
retryDuringWave: values["mix-retry-during-wave"] === true,
settleMinutes: whole(values["mix-settle-minutes"], "mix-settle-minutes", 15, 0, 120),
...(values["mix-tasks"] !== undefined ? { tasks: whole(values["mix-tasks"], "mix-tasks", 1, 1, 2_000) } : {}),
};
if (mix.tasks === undefined && mix.arrivalEndMs < (mix.runs + mix.queue) * BURST_GAP_MS) {
@@ -269,7 +281,8 @@ export function planStagingMixTasks(options: WaveOptions, rng = createRng((optio
holdMs: 0,
followUpAfterMs: null,
nonce: rng.hex(8),
stagingMix: { turnMinutes, rounds, subtasks, followUpAfterMs, arriveAtMs, holdMs: mix.holdMinutes * 60_000 },
stagingMix: { turnMinutes, rounds, subtasks, followUpAfterMs, arriveAtMs, holdMs: mix.holdMinutes * 60_000,
...(mix.retryDuringWave ? { retryDuringWave: true } : {}) },
};
});
}
@@ -354,6 +367,10 @@ export function stagingMixDescription(nonce: string, task: StagingMixTask, agent
export const followUpRequestId = (issueId: string, nonce: string) => stableRequestId(`followup-${issueId}-${nonce}`);
export function isDoneComment(body: string, nonce: string): boolean {
return body.trim().replace(/\.$/, "").replace(/^`+|`+$/g, "").trim() === doneMarker(nonce);
}
export function followUpComment(nonce: string): string {
return `Additional requirement: include the line \`${followUpMarker(nonce)}\` on its own line in this task's \`ledger\` document. Keep going with your rounds; do not restart any.`;
}
@@ -387,7 +404,9 @@ export function verifyLedger(nonce: string, rounds: number, text: string | null
export function gradeStagingMix(nonce: string, task: StagingMixTask, artifacts: OracleArtifacts): OracleResult {
const ledger = artifacts.documents.ledger ?? null;
const own = verifyLedger(nonce, task.rounds, ledger);
const doneComments = artifacts.agentComments.filter(body => body.includes(doneMarker(nonce))).length;
const comments = artifacts.stagingMix?.comments ?? artifacts.agentComments.map(body => ({ body, runId: null }));
const doneComments = comments.filter(comment => isDoneComment(comment.body, nonce)).length;
const completionRuns = [...new Set(comments.filter(comment => comment.body.includes(doneMarker(nonce)) && comment.runId).map(comment => comment.runId!))];
const followUp = artifacts.stagingMix?.followUp ?? null;
const children = (artifacts.children ?? []).filter(child => (child.title ?? "").includes(nonce));
const checks: OracleResult["checks"] = [
@@ -395,6 +414,8 @@ export function gradeStagingMix(nonce: string, task: StagingMixTask, artifacts:
{ name: "ledger_rounds_verified", ok: own.verified, detail: own.detail },
{ name: "rounds_once", ok: own.duplicates === 0, detail: `duplicates=${own.duplicates}` },
{ name: "done_comment_once", ok: doneComments === 1, detail: `comments=${doneComments}` },
{ name: "completion_once", ok: completionRuns.length <= 1,
detail: completionRuns.length > 1 ? `duplicate completion: runs ${completionRuns.map(id => id.slice(0, 8)).join(", ")} each reported completion` : `runs=${completionRuns.length}` },
];
if (task.followUpAfterMs !== null) {
const errors = artifacts.stagingMix?.followUpErrors ?? [];
@@ -502,17 +523,46 @@ export async function stagingMixStep(input: StagingMixStepInput): Promise<void>
input.note("staging_mix_follow_up", { ...followUp });
}
}
noteRetryBanner(state, retryShown(sample), now, latestFailureCode(sample));
if (!retryOffered(sample) || !retryDue(record)) {
if (!retryOffered(sample)) state.retrySeenAt = null;
return;
}
state.retrySeenAt ??= now;
if (!overlap) return;
if (!overlap || !task.retryDuringWave) return;
(state.retries ??= []).push({ at: now, inOverlap: true });
input.note("staging_mix_retry_in_overlap", { waitedMs: now - state.retrySeenAt });
await input.retry(sample, now);
}
export function stagingMixArtifacts(record: TaskRecord): StagingMixArtifacts {
return { followUp: record.stagingMix?.followUp ?? null, followUpErrors: record.stagingMix?.followUpErrors ?? [], retries: record.stagingMix?.retries ?? [] };
export function retryShown(sample: Pick<TaskSample, "reachable" | "issueStatus" | "interruption">): boolean {
return sample.reachable && sample.issueStatus !== "done" && Boolean(sample.interruption?.canRetry);
}
export function latestFailureCode(sample: Pick<TaskSample, "runs">): string | null {
const failed = (sample.runs ?? []).filter(run => run.status === "failed" || run.status === "timed_out" || run.errorCode)
.sort((a, b) => Date.parse(b.createdAt ?? "") - Date.parse(a.createdAt ?? ""));
return failed[0]?.errorCode ?? failed[0]?.failureCause ?? null;
}
export function noteRetryBanner(state: StagingMixRecord, shown: boolean, now: number, errorCode: string | null): void {
const banner = state.retryBanner;
if (!shown) {
if (banner?.visibleSince != null) { banner.shownMs += now - banner.visibleSince; banner.visibleSince = null; }
return;
}
if (!banner) { state.retryBanner = { firstShownAt: now, lastShownAt: now, shownMs: 0, visibleSince: now, errorCode }; return; }
banner.lastShownAt = now;
banner.visibleSince ??= now;
banner.errorCode = errorCode ?? banner.errorCode;
}
export function bannerShownMs(banner: RetryBanner, now: number): number {
return banner.shownMs + (banner.visibleSince === null ? 0 : now - banner.visibleSince);
}
export async function collectStagingMixArtifacts(api: Pick<ChaosApi, "get">, issueId: string, record: TaskRecord): Promise<StagingMixArtifacts> {
const rows = await api.get<Array<{ body?: string; authorAgentId?: string | null; createdByRunId?: string | null }>>(`/api/issues/${issueId}/comments`).catch(() => null);
return { followUp: record.stagingMix?.followUp ?? null, followUpErrors: record.stagingMix?.followUpErrors ?? [], retries: record.stagingMix?.retries ?? [],
...(rows ? { comments: rows.filter(row => row.authorAgentId).map(row => ({ body: String(row.body ?? ""), runId: row.createdByRunId ?? null })) } : {}) };
}