test(restart-chaos): count hands-off completion from the deliverable and never click a scheduled retry

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-12 12:53:40 -06:00
1 parent ddb64d3223
commit 59b1b37870
9 files changed
+228 -92

No files matched your search

+32 -10
View File
@@ -1397,15 +1397,35 @@ own continuation is what gets measured. A task that shows a Retry banner is kept
is recorded per task: when it first showed, how long it stayed, and the error code behind it, even when the task then
finished by itself. A banner is a defect, because the product goal is that it never appears. After the settle
(`--mix-settle-minutes`, default 15, after the last fault slot ends), each task that is still not done and still
offers Retry gets one click (`staging_mix_retry_after_settle`), and the harness follows it to rest. When the release-swap
cell drives the wave, its own end-of-wave pass makes that one click instead. Both paths use
`shouldClickRetryAfterSettle` in `staging-mix-retry.ts`: never on a done task, a task waiting on a scheduled retry, or a
task that was already clicked. So no task gets two clicks. `--mix-retry-during-wave` restores the old behaviour: Retries
offers Retry gets one click (`staging_mix_retry_after_settle`). The harness follows the click to rest for up to 15 minutes,
whatever the settle setting. A task that finished by itself during the settle is watched again until it rests, so its
record is fresh before grading. When the release-swap cell drives the wave, its own end-of-wave pass makes that one
click instead, and its issue reads also update the banner record. Both paths use `shouldClickRetryAfterSettle` in
`staging-mix-retry.ts`, which never clicks:
- a done task;
- a task with a scheduled retry (`scheduledRetry`, state `retry_scheduled` or a `nextRetryAt`);
- a task that was already clicked.
So no task gets two clicks. `--mix-retry-during-wave` restores the old behaviour: Retries
clicked during the wave, held until an overlap window as follow-ups are (`staging_mix_retry_in_overlap`).
**Results.** `staging-mix-results.json` and the first lines of `SUMMARY.md` give three numbers, in this order:
`completed hands-off / started`, `Retry banner shown: N tasks` and `Retry worked / Retry clicked`. A Retry counts as
worked when the task is done at the end. The banner list follows, then one read of
**Results.** `staging-mix-results.json` and the first lines of `SUMMARY.md` give four lines, in this order:
1. `completed hands-off / started`.
2. `duplicated side effects: N tasks`.
3. `Retry banner shown: N tasks`.
4. `Retry worked / Retry clicked`.
A task counts as completed hands-off when all three hold, read from fresh data at the end of the wave:
- its status is `done`;
- every check of its deliverable passes (ledger rounds, DONE comment posted, follow-up delivered, subtasks);
- nobody clicked Retry.
A duplicated side effect (`rounds_once`, `done_comment_once`, `completion_once`, `subtasks_once`,
`subtask_rounds_once`) is counted on the second line. It does not remove the task from the first line. A Retry counts as
worked when the clicked task meets the same completion rule. The banner list follows, then one read of
`GET /api/companies/:id/assigned-work-invariant` at the end of the wave: `violationCount`, `violations` by reason,
`agentsWithStalledSlots` and `openTasks`.
@@ -1417,11 +1437,13 @@ not the status:
| `issue_done` | the task is not `done` |
| `ledger_rounds_verified` | a round is missing, ran under 80 s, or its hash differs from the harness's own computation (POSIX `cksum`, chained per tick) |
| `rounds_once` | a round line appears twice (a tool ran again) |
| `done_comment_once` | the comment whose whole body is `DONE-<nonce>` is missing or was posted twice; a summary that quotes the marker does not count |
| `completion_once` | more than one run posted a comment that mentions `DONE-<nonce>` (a duplicate completion) |
| `done_comment_posted` | no comment's whole body is `DONE-<nonce>`; a summary that quotes the marker does not count |
| `done_comment_once` | that comment was posted more than once |
| `completion_once` | more than one run posted a comment that mentions `DONE-<nonce>` (a duplicate completion), or the comments could not be read |
| `follow_up_posted` | a planned follow-up was never posted (the detail lists the post errors) |
| `follow_up_served` | a follow-up was posted during a run and the ledger lacks its line |
| `subtasks_once`, `subtask_<k>_done`, `parent_read_subtask_<k>` | a parent created more or fewer than two subtasks, a subtask's ledger fails, or the parent did not report a subtask's last hash |
| `subtask_<k>_done`, `parent_read_subtask_<k>` | a subtask is missing, not done or its ledger fails, or the parent did not report its last hash |
| `subtasks_once`, `subtask_rounds_once` | a parent created more than two subtasks, or a subtask repeated a round |
Duplicated side effects (a repeated round, comment or subtask) fail the task and show in the signature.
+1 -1
View File
@@ -836,7 +836,7 @@ async function main(): Promise<number> {
rules: Object.fromEntries(Object.entries(grade.rules).map(([id, rule]) => [id, rule.verdict])), T: grade.T.verdict,
universal: Object.fromEntries(Object.entries(grade.universal ?? {}).map(([id, rule]) => [id, `${rule.verdict}${rule.enforced ? "" : " (warning)"}`])) });
}
if (mix) await writeStagingMixResults({ waveDir, api, companyId: lab.companyId, tasks: launched, passed: new Map(lines.map(line => [line.taskId, line.pass])), timeline: timeline.entries })
if (mix) await writeStagingMixResults({ waveDir, api, companyId: lab.companyId, tasks: launched, timeline: timeline.entries })
.catch(error => { harnessError ??= new Error(`Staging-mix results failed: ${String(error)}`); });
// MS1–MS3: the cross-server invariant over the whole wave (multi-server.ts).
if (faults.some(receipt => receipt.multiServer)) {
+8 -3
View File
@@ -698,10 +698,15 @@ export class TaskObserver {
canRetry: sample.interruption?.canRetry ?? false, retried: Boolean(this.record.retry) });
}
async retryAfterSettle(sample: TaskSample, now: number): Promise<void> {
reopen(now: number): void {
this.done = false;
this.record.restAt = null;
this.record.timedOut = false;
this.record.deadlineAt = now + RETRY_BUDGET_MS;
}
async retryAfterSettle(sample: TaskSample, now: number): Promise<void> {
this.reopen(now);
await this.boardRetry(sample, now);
this.record.deadlineAt = now + RETRY_BUDGET_MS;
}
@@ -1043,7 +1048,7 @@ export class TaskObserver {
const document = await this.api.get<Json>(`/api/issues/${issueId}/documents/${key}`).catch(() => null);
documents[key] = typeof document?.body === "string" ? document.body : null;
}
const comments = await this.api.get<Json[]>(`/api/issues/${issueId}/comments`).catch(() => []);
const comments = await this.api.get<Json[]>(`/api/issues/${issueId}/comments`).catch(() => null);
const workspacePath = await this.workspacePath();
const snapshotPath = this.record.preFaultWorkspacePath ?? workspacePath;
if (snapshotPath && this.record.preFaultFiles) {
@@ -1101,7 +1106,7 @@ export class TaskObserver {
status: String(run.status), errorCode: run.errorCode ?? null, createdAt: run.createdAt ?? null, finishedAt: run.finishedAt ?? null }))).catch(() => []),
...(this.journey ? { journey: await this.journey.collect(issueId).finally(() => this.journey!.close()) } : {}),
...(this.plan.workload === "H2" ? { handoff: await this.collectHandoff(issueId, issue) } : {}),
...(this.plan.stagingMix ? { stagingMix: await collectStagingMixArtifacts(this.api, issueId, this.record) } : {}),
...(this.plan.stagingMix ? { stagingMix: collectStagingMixArtifacts(this.record, comments) } : {}),
};
}
@@ -31,6 +31,7 @@ export interface ReleaseSwapFinishInput {
record: (kind: string, data: Record<string, unknown>) => void;
pollMs?: number;
drainCapMs?: number;
onView?: (issueId: string, issue: Json | null, at: number) => void;
}
type Client = Pick<Db, "$client">;
@@ -200,6 +201,7 @@ export async function finishReleaseSwap(input: ReleaseSwapFinishInput): Promise<
const left = deadline - Date.now();
views.set(id, left <= 0 ? { status: "unreadable", error: "view deadline passed" }
: await input.api.get<Json>(`/api/issues/${id}`, Math.min(10_000, left)).catch(error => ({ status: "unreadable", error: String(error) })));
input.onView?.(id, views.get(id)!, Date.now());
}
}));
};
+3 -1
View File
@@ -7,6 +7,7 @@ import { ReleaseSwapRun, overlapLaunchExtensionMs, type ReleaseSwapOptions } fro
import { finishReleaseSwap, type ReleaseSwapFinishInput } from "./release-swap-finish.js";
import { StormRun } from "./scheduler.js";
import { setOverlapProbe } from "./staging-mix.js";
import { bannerSink } from "./staging-mix-retry.js";
import type { ChaosSupervisor } from "./supervisor.js";
import type { TaskObserver } from "./observer.js";
import { waitForAgentsReady, type ChaosLab } from "./seed.js";
@@ -87,7 +88,8 @@ export class ReleaseSwapWave {
const graded = await finishReleaseSwap({ driver: this.driver, settleMs: options.settleMs, companyId: this.lab.companyId, api: this.api, db: this.db, board: input.board,
drainCapMs: options.drainMs ?? Math.max(0, ...input.observers.map(observer => observer.definition.budgetMs)) + options.settleMs,
timeline: this.timeline.entries, issueIdForTask: new Map(input.launched.flatMap(observer => observer.record.issueId ? [[observer.plan.taskId, observer.record.issueId] as const] : [])),
serverWindows: this.supervisor.serverWindows, waveDir: input.waveDir, cancelled: this.cancelled, record: (kind, data) => { this.timeline.add(kind, data); } });
serverWindows: this.supervisor.serverWindows, waveDir: input.waveDir, cancelled: this.cancelled, record: (kind, data) => { this.timeline.add(kind, data); },
onView: bannerSink(input.launched) });
return graded.pass;
} finally {
setOverlapProbe(null);
+88 -38
View File
@@ -1,8 +1,9 @@
import { describe, expect, it } from "vitest";
import { parseCommand } from "./config.js";
import { doneMarker, expectedRoundHash, gradeStagingMix, planStagingMixWave, ROUND_SECONDS, type StagingMixTask } from "./staging-mix.js";
import { collectStagingMixArtifacts, doneMarker, expectedRoundHash, gradeStagingMix, planStagingMixWave, ROUND_SECONDS, type StagingMixTask } from "./staging-mix.js";
import {
readAssignedWorkInvariant, retryAfterSettleStage, shouldClickRetryAfterSettle, stagingMixHeadline, summarizeStagingMix, type RetryStageObserver,
bannerSink, issueView, readAssignedWorkInvariant, retryAfterSettleStage, shouldClickRetryAfterSettle, stagingMixHeadline, summarizeStagingMix,
type RetryStageObserver,
} from "./staging-mix-retry.js";
import type { OracleArtifacts, TaskPlan, TaskRecord, TaskSample } from "./types.js";
@@ -30,63 +31,105 @@ describe("staging-mix Retry policy", () => {
expect(planStagingMixWave("w", old).tasks.every(entry => entry.stagingMix!.retryDuringWave === true)).toBe(true);
});
it("clicks once after the settle, only on a task still offering Retry that nobody clicked", () => {
expect(shouldClickRetryAfterSettle({ status: "blocked", canRetry: true, retryWait: false }, 0)).toBe(true);
expect(shouldClickRetryAfterSettle({ status: "blocked", canRetry: true, retryWait: false }, 1)).toBe(false);
expect(shouldClickRetryAfterSettle({ status: "done", canRetry: true, retryWait: false }, 0)).toBe(false);
expect(shouldClickRetryAfterSettle({ status: "blocked", canRetry: true, retryWait: true }, 0)).toBe(false);
expect(shouldClickRetryAfterSettle({ status: "in_progress", canRetry: false, retryWait: false }, 0)).toBe(false);
it("clicks once after the settle, only on a task still offering Retry that nobody clicked and that has no retry scheduled", () => {
const view = (overrides: Partial<ReturnType<typeof issueView>> = {}) => ({ status: "blocked", canRetry: true, retryWait: false, retryScheduled: false, ...overrides });
expect(shouldClickRetryAfterSettle(view(), 0)).toBe(true);
expect(shouldClickRetryAfterSettle(view(), 1)).toBe(false);
expect(shouldClickRetryAfterSettle(view({ status: "done" }), 0)).toBe(false);
expect(shouldClickRetryAfterSettle(view({ retryWait: true }), 0)).toBe(false);
expect(shouldClickRetryAfterSettle(view({ canRetry: false }), 0)).toBe(false);
expect(shouldClickRetryAfterSettle(view({ retryScheduled: true }), 0)).toBe(false);
const scheduled = issueView({ status: "in_progress", interruption: { state: "retry_scheduled", canRetry: true, nextRetryAt: "2026-11-12T20:00:00Z" } });
expect(scheduled.retryScheduled).toBe(true);
expect(shouldClickRetryAfterSettle(scheduled, 0)).toBe(false);
expect(issueView({ status: "blocked", scheduledRetry: { scheduledRetryAt: "x" }, interruption: { state: "failed", canRetry: true } }).retryScheduled).toBe(true);
expect(issueView({ status: "blocked", interruption: { state: "failed", canRetry: true } }).retryScheduled).toBe(false);
});
it("waits out the settle hands-off, then clicks once per stuck task and follows it to rest", async () => {
const harness = () => {
let clock = 0;
const issues: Record<string, Record<string, unknown>> = {
i1: { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r1" } },
i2: { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r2" } },
i3: { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r3" } },
};
const observer = (taskId: string, issueId: string, retry: TaskRecord["retry"] = null): RetryStageObserver & { clicks: number } => {
const issues: Record<string, Record<string, unknown>> = {};
const observer = (taskId: string, issueId: string, overrides: Partial<TaskRecord> = {}, finishAfterClickMs = 0) => {
const self = {
plan: { taskId, stagingMix: task() } as TaskPlan, record: { issueId, retry } as TaskRecord, done: true, clicks: 0,
async poll() { if (issues[issueId]!.status === "done") self.done = true; },
plan: { taskId, stagingMix: task() } as TaskPlan, record: { issueId, retry: null, finalStatus: "blocked", ...overrides } as TaskRecord, done: true,
clicks: 0, reopened: 0, clickedAt: null as number | null,
async poll() {
if (issues[issueId]!.status === "done" || (self.clickedAt !== null && clock - self.clickedAt >= finishAfterClickMs)) { issues[issueId] = { status: "done" }; self.done = true; }
},
reopen() { self.done = false; self.reopened += 1; },
async retryAfterSettle(sample: TaskSample, at: number) {
self.done = false; self.clicks += 1;
self.done = false; self.clicks += 1; self.clickedAt = at;
expect(sample.interruption?.observedRunId).toBe(`r${issueId.slice(1)}`);
self.record.retry = { at, accepted: true } as TaskRecord["retry"];
issues[issueId] = { status: "done" };
},
};
return self;
return self satisfies RetryStageObserver;
};
const healsItself = observer("t1", "i1"), stuck = observer("t2", "i2"), alreadyClicked = observer("t3", "i3", { at: 1, accepted: true } as TaskRecord["retry"]);
return { issues, observer, now: () => clock, advance: (ms: number) => { clock += ms; } };
};
it("waits out the settle hands-off, refreshes a task that healed, clicks once per stuck task and follows it on the retry budget", async () => {
const h = harness();
const failed = (id: string) => ({ status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: `r${id}` } });
Object.assign(h.issues, { i1: failed("1"), i2: failed("2"), i3: failed("3"), i4: { status: "in_progress", interruption: { state: "retry_scheduled", canRetry: true } } });
const healsItself = h.observer("t1", "i1"), stuck = h.observer("t2", "i2", {}, 10 * 60_000);
const alreadyClicked = h.observer("t3", "i3", { retry: { at: 1, accepted: true } as TaskRecord["retry"] }), scheduled = h.observer("t4", "i4");
const notes: string[] = [];
const result = await retryAfterSettleStage({ observers: [healsItself, stuck, alreadyClicked], api: { get: async <T>(path: string) => issues[path.split("/").pop()!] as T },
settleUntil: 120_000, settleMs: 60_000, cancelled: () => false, note: kind => notes.push(kind), pollMs: 60_000, now: () => clock,
sleep: async ms => { clock += ms; if (clock >= 60_000) issues.i1 = { status: "done" }; } });
expect(clock).toBeGreaterThanOrEqual(120_000);
expect(result.clicked).toEqual(["t2"]);
expect([healsItself.clicks, stuck.clicks, alreadyClicked.clicks]).toEqual([0, 1, 0]);
expect(stuck.done).toBe(true);
const result = await retryAfterSettleStage({ observers: [healsItself, stuck, alreadyClicked, scheduled],
api: { get: async <T>(path: string) => h.issues[path.split("/").pop()!] as T }, settleUntil: 120_000, cancelled: () => false,
note: kind => notes.push(kind), pollMs: 60_000, now: h.now, sleep: async ms => { h.advance(ms); if (h.now() >= 60_000) h.issues.i1 = { status: "done" }; } });
expect(result).toEqual({ clicked: ["t2"], refreshed: ["t1"] });
expect([healsItself.clicks, stuck.clicks, alreadyClicked.clicks, scheduled.clicks]).toEqual([0, 1, 0, 0]);
expect(healsItself.reopened).toBe(1);
expect([healsItself.done, stuck.done]).toEqual([true, true]);
expect(healsItself.record.stagingMix!.retryBanner).toMatchObject({ firstShownAt: 0, shownMs: 60_000 });
expect(stuck.record.stagingMix!.retryAfterSettle).toMatchObject({ accepted: true, issueStatusAtClick: "blocked" });
expect(notes).toEqual(["staging_mix_settling", "staging_mix_retry_after_settle"]);
expect(notes).toEqual(["staging_mix_settling", "staging_mix_retry_after_settle", "staging_mix_refreshed_after_settle"]);
});
it("reports the three numbers first, counting the swap grader's own clicks and banners that healed by themselves", () => {
it("follows a click on the retry budget even with a zero settle", async () => {
const h = harness();
h.issues.i2 = { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r2" } };
const slow = h.observer("t2", "i2", {}, 12 * 60_000);
await retryAfterSettleStage({ observers: [slow], api: { get: async <T>(path: string) => h.issues[path.split("/").pop()!] as T }, settleUntil: 0,
cancelled: () => false, note: () => undefined, now: h.now, sleep: async ms => { h.advance(ms); } });
expect(slow.clicks).toBe(1);
expect(slow.done).toBe(true);
expect(h.now()).toBeGreaterThanOrEqual(12 * 60_000);
});
it("feeds the swap grader's reads into banner tracking", () => {
const record = { issueId: "i1" } as TaskRecord;
const other = { issueId: "i9" } as TaskRecord;
const sink = bannerSink([{ plan: { stagingMix: task() } as TaskPlan, record }, { plan: {} as TaskPlan, record: other }]);
sink("i1", { status: "blocked", interruption: { canRetry: true } }, 1_000);
sink("i1", { status: "unreadable" }, 2_000);
sink("i1", { status: "in_progress", interruption: null }, 31_000);
sink("i9", { status: "blocked", interruption: { canRetry: true } }, 1_000);
expect(record.stagingMix!.retryBanner).toMatchObject({ firstShownAt: 1_000, shownMs: 30_000, visibleSince: null });
expect(other.stagingMix).toBeUndefined();
});
it("counts done tasks with a verified deliverable and no click, and reports duplicates on their own line", () => {
const record = (overrides: Partial<TaskRecord> = {}) => ({ issueId: null, retry: null, ...overrides }) as TaskRecord;
const banner = { firstShownAt: 0, lastShownAt: 30_000, shownMs: 30_000, visibleSince: null, errorCode: "controller_lease_lost" };
const ok = (name: string) => ({ name, ok: true }), bad = (name: string) => ({ name, ok: false });
const clean = [ok("issue_done"), ok("ledger_rounds_verified"), ok("done_comment_posted"), ok("rounds_once"), ok("completion_once")];
const result = summarizeStagingMix([
{ taskId: "t1", issueId: "i1", record: record(), pass: true, finalStatus: "done" },
{ taskId: "t2", issueId: "i2", record: record({ stagingMix: { retryBanner: banner } }), pass: true, finalStatus: "done" },
{ taskId: "t3", issueId: "i3", record: record({ retry: { at: 1, accepted: true } as TaskRecord["retry"], stagingMix: { retryBanner: banner } }), pass: false, finalStatus: "done" },
{ taskId: "t4", issueId: "i4", record: record({ stagingMix: { retryBanner: banner } }), pass: false, finalStatus: "blocked" },
{ taskId: "t5", issueId: "i5", record: record(), pass: false, finalStatus: "in_progress" },
], new Map([["i4", true]]), { violationCount: 2, violations: { nothing_pending: 2 }, agentsWithStalledSlots: 1, openTasks: 3 }, 60_000);
{ taskId: "t1", issueId: "i1", record: record({ finalStatus: "blocked", timedOut: true }), finalStatus: "done", checks: clean },
{ taskId: "t2", issueId: "i2", record: record({ stagingMix: { retryBanner: banner } }), finalStatus: "done", checks: [...clean.slice(0, 4), bad("completion_once")] },
{ taskId: "t3", issueId: "i3", record: record({ retry: { at: 1, accepted: true } as TaskRecord["retry"], stagingMix: { retryBanner: banner } }), finalStatus: "done", checks: clean },
{ taskId: "t4", issueId: "i4", record: record({ stagingMix: { retryBanner: banner } }), finalStatus: "blocked", checks: [bad("issue_done")] },
{ taskId: "t5", issueId: "i5", record: record(), finalStatus: "done", checks: [ok("issue_done"), bad("ledger_rounds_verified")] },
{ taskId: "t6", issueId: "i6", record: record(), finalStatus: "done", checks: null },
], new Set(["i4"]), { violationCount: 2, violations: { nothing_pending: 2 }, agentsWithStalledSlots: 1, openTasks: 3 }, 60_000);
expect(stagingMixHeadline(result)).toEqual([
"completed hands-off / started: 2 / 5",
"completed hands-off / started: 2 / 6",
"duplicated side effects: 1 tasks",
"Retry banner shown: 3 tasks",
"Retry worked / Retry clicked: 1 / 2",
]);
expect(result.duplicatedSideEffects).toEqual([{ taskId: "t2", checks: ["completion_once"] }]);
expect(result.bannerShown.find(entry => entry.taskId === "t2")).toMatchObject({ completedAnyway: true, errorCode: "controller_lease_lost", shownMs: 30_000 });
expect(result.invariant.violationCount).toBe(2);
});
@@ -103,7 +146,7 @@ describe("staging-mix completion comments", () => {
const nonce = "b3d70e77";
const base = (comments: Array<{ body: string; runId: string | null }>): OracleArtifacts => ({ issueStatus: "done", documents: { ledger: ledger(nonce, 4) },
agentComments: comments.map(comment => comment.body), files: {}, commands: {}, workspacePath: null,
stagingMix: { followUp: null, followUpErrors: [], retries: [], comments } });
stagingMix: collectStagingMixArtifacts({} as TaskRecord, comments.map(comment => ({ body: comment.body, authorAgentId: "a", createdByRunId: comment.runId }))) });
const check = (artifacts: OracleArtifacts, name: string) => gradeStagingMix(nonce, task(), artifacts).checks.find(entry => entry.name === name)!;
it("counts only the DONE comment itself, not a closing summary that quotes it", () => {
@@ -116,6 +159,13 @@ describe("staging-mix completion comments", () => {
expect(check(base(comments), "completion_once").ok).toBe(true);
expect(gradeStagingMix(nonce, task(), base(comments)).verdict).toBe("pass");
expect(check(base([...comments, { body: ` \`${doneMarker(nonce)}\`. `, runId: "run-c" }]), "done_comment_once")).toMatchObject({ ok: false, detail: "comments=2" });
expect(check(base(comments.slice(0, 2)), "done_comment_posted")).toMatchObject({ ok: false, detail: "comments=0" });
});
it("marks completion_once unreadable instead of passing when the comments read failed", () => {
const unread: OracleArtifacts = { ...base([{ body: doneMarker(nonce), runId: "run-b" }]), stagingMix: collectStagingMixArtifacts({} as TaskRecord, null) };
expect(check(unread, "completion_once")).toMatchObject({ ok: false, detail: "unreadable: the task's comments could not be read" });
expect(check(unread, "done_comment_posted").ok).toBe(true);
});
it("flags a completion summary repeated by a different run as a duplicate completion", () => {
+74 -28
View File
@@ -1,16 +1,25 @@
import { writeFile } from "node:fs/promises";
import { readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import type { ChaosApi } from "./api.js";
import { bannerShownMs, noteRetryBanner } from "./staging-mix.js";
import type { InterruptionSample, TaskPlan, TaskRecord, TaskSample } from "./types.js";
import { bannerShownMs, DUPLICATE_CHECKS, noteRetryBanner } from "./staging-mix.js";
import type { InterruptionSample, OracleResult, TaskPlan, TaskRecord, TaskSample } from "./types.js";
type Json = Record<string, any>;
export interface IssueView { status: string; canRetry: boolean; retryWait: boolean; observedRunId: string | null; interruption: InterruptionSample | null }
export const RETRY_FOLLOW_MS = 15 * 60_000;
export function shouldClickRetryAfterSettle(view: Pick<IssueView, "status" | "canRetry" | "retryWait">, priorClicks: number): boolean {
return view.status !== "done" && view.canRetry && !view.retryWait && priorClicks === 0;
export interface IssueView {
status: string;
canRetry: boolean;
retryWait: boolean;
retryScheduled: boolean;
observedRunId: string | null;
interruption: InterruptionSample | null;
}
export function shouldClickRetryAfterSettle(view: Pick<IssueView, "status" | "canRetry" | "retryWait" | "retryScheduled">, priorClicks: number): boolean {
return view.status !== "done" && view.canRetry && !view.retryWait && !view.retryScheduled && priorClicks === 0;
}
export function issueView(issue: Json | null): IssueView {
@@ -19,6 +28,7 @@ export function issueView(issue: Json | null): IssueView {
status: String(issue?.status ?? "unreadable"),
canRetry: interruption?.canRetry === true,
retryWait: Boolean(interruption?.retryWait),
retryScheduled: Boolean(issue?.scheduledRetry) || interruption?.state === "retry_scheduled" || Boolean(interruption?.nextRetryAt),
observedRunId: typeof interruption?.observedRunId === "string" ? interruption.observedRunId : null,
interruption: interruption ? { state: String(interruption.state), canRetry: interruption.canRetry === true, retryWait: Boolean(interruption.retryWait),
workspaceFailure: null, observedRunId: typeof interruption.observedRunId === "string" ? interruption.observedRunId : null,
@@ -26,11 +36,26 @@ export function issueView(issue: Json | null): IssueView {
};
}
export function noteBannerFromIssue(record: TaskRecord, issue: Json | null, at: number): void {
if (!issue || issue.status === "unreadable") return;
const view = issueView(issue);
noteRetryBanner(record.stagingMix ??= {}, view.status !== "done" && view.canRetry, at, null);
}
export function bannerSink(observers: ReadonlyArray<{ plan: TaskPlan; record: TaskRecord }>): (issueId: string, issue: Json | null, at: number) => void {
const byIssue = new Map(observers.flatMap(observer => observer.plan.stagingMix && observer.record.issueId ? [[observer.record.issueId, observer] as const] : []));
return (issueId, issue, at) => {
const observer = byIssue.get(issueId);
if (observer) noteBannerFromIssue(observer.record, issue, at);
};
}
export interface RetryStageObserver {
plan: TaskPlan;
record: TaskRecord;
done: boolean;
poll(now?: number): Promise<void>;
reopen(now: number): void;
retryAfterSettle(sample: TaskSample, now: number): Promise<void>;
}
@@ -38,15 +63,15 @@ export interface RetryStageInput {
observers: readonly RetryStageObserver[];
api: Pick<ChaosApi, "get">;
settleUntil: number;
settleMs: number;
cancelled: () => boolean;
note: (kind: string, data: Record<string, unknown>) => void;
followMs?: number;
pollMs?: number;
now?: () => number;
sleep?: (ms: number) => Promise<void>;
}
export async function retryAfterSettleStage(input: RetryStageInput): Promise<{ clicked: string[] }> {
export async function retryAfterSettleStage(input: RetryStageInput): Promise<{ clicked: string[]; refreshed: string[] }> {
const now = input.now ?? Date.now;
const sleep = input.sleep ?? (ms => delay(ms));
const pollMs = input.pollMs ?? 60_000;
@@ -55,9 +80,8 @@ export async function retryAfterSettleStage(input: RetryStageInput): Promise<{ c
const read = async (observers: readonly RetryStageObserver[]) => {
for (const observer of observers) {
const issue = await input.api.get<Json>(`/api/issues/${observer.record.issueId}`, 10_000).catch(() => null);
const view = issueView(issue);
views.set(observer, view);
noteRetryBanner(observer.record.stagingMix ??= {}, view.status !== "done" && view.canRetry, now(), null);
views.set(observer, issueView(issue));
noteBannerFromIssue(observer.record, issue, now());
}
};
input.note("staging_mix_settling", { settleUntil: new Date(input.settleUntil).toISOString(), tasks: tracked.length });
@@ -66,46 +90,66 @@ export async function retryAfterSettleStage(input: RetryStageInput): Promise<{ c
if (input.cancelled() || now() >= input.settleUntil || tracked.every(observer => views.get(observer)?.status === "done")) break;
await sleep(Math.min(pollMs, Math.max(1, input.settleUntil - now())));
}
const clicked: RetryStageObserver[] = [];
const followed: RetryStageObserver[] = [];
const clicked: string[] = [], refreshed: string[] = [];
for (const observer of tracked) {
const view = views.get(observer);
if (!view || input.cancelled() || !shouldClickRetryAfterSettle(view, observer.record.retry ? 1 : 0)) continue;
if (!view || input.cancelled()) continue;
if (view.status === "done" && observer.record.finalStatus !== "done") {
observer.reopen(now());
refreshed.push(observer.plan.taskId);
followed.push(observer);
continue;
}
if (!shouldClickRetryAfterSettle(view, observer.record.retry ? 1 : 0)) continue;
const at = now();
await observer.retryAfterSettle({ at, reachable: true, serverReady: true, issueStatus: view.status, interruption: view.interruption }, at);
(observer.record.stagingMix ??= {}).retryAfterSettle = { at, accepted: observer.record.retry?.accepted ?? null, issueStatusAtClick: view.status };
input.note("staging_mix_retry_after_settle", { taskId: observer.plan.taskId, accepted: observer.record.retry?.accepted ?? null });
clicked.push(observer);
clicked.push(observer.plan.taskId);
followed.push(observer);
}
const until = now() + input.settleMs;
while (clicked.some(observer => !observer.done) && !input.cancelled() && now() < until) {
for (const observer of clicked) if (!observer.done) await observer.poll(now());
if (refreshed.length) input.note("staging_mix_refreshed_after_settle", { taskIds: refreshed });
const until = now() + (input.followMs ?? RETRY_FOLLOW_MS);
while (followed.some(observer => !observer.done) && !input.cancelled() && now() < until) {
for (const observer of followed) if (!observer.done) await observer.poll(now());
await sleep(Math.min(pollMs, 15_000));
}
return { clicked: clicked.map(observer => observer.plan.taskId) };
return { clicked, refreshed };
}
export interface ResultTask { taskId: string; issueId: string | null; record: TaskRecord; pass: boolean; finalStatus: string | null }
export interface ResultTask { taskId: string; issueId: string | null; record: TaskRecord; finalStatus: string | null; checks: OracleResult["checks"] | null }
export interface StagingMixResult {
started: number;
completedHandsOff: number;
duplicatedSideEffects: Array<{ taskId: string; checks: string[] }>;
bannerShown: Array<{ taskId: string; firstShownAt: string; shownMs: number; errorCode: string | null; completedAnyway: boolean }>;
retryClicked: number;
retryWorked: number;
invariant: { violationCount: number | null; violations: Record<string, number> | null; agentsWithStalledSlots: number | null; openTasks: number | null; error?: string };
}
export function summarizeStagingMix(tasks: readonly ResultTask[], extraClicks: ReadonlyMap<string, boolean | null>, invariant: StagingMixResult["invariant"], now = Date.now()): StagingMixResult {
export function deliverableVerified(checks: OracleResult["checks"] | null): boolean {
return Boolean(checks?.length) && checks!.every(check => check.ok || DUPLICATE_CHECKS.has(check.name));
}
export function summarizeStagingMix(tasks: readonly ResultTask[], extraClicks: ReadonlySet<string>, invariant: StagingMixResult["invariant"], now = Date.now()): StagingMixResult {
const clicked = (task: ResultTask) => Boolean(task.record.retry || task.record.priorRetries?.length || (task.issueId && extraClicks.has(task.issueId)));
const completed = (task: ResultTask) => task.finalStatus === "done" && deliverableVerified(task.checks);
const clickedTasks = tasks.filter(clicked);
return {
started: tasks.length,
completedHandsOff: tasks.filter(task => task.pass && !clicked(task)).length,
completedHandsOff: tasks.filter(task => completed(task) && !clicked(task)).length,
duplicatedSideEffects: tasks.flatMap(task => {
const failing = (task.checks ?? []).filter(check => !check.ok && DUPLICATE_CHECKS.has(check.name)).map(check => check.name);
return failing.length ? [{ taskId: task.taskId, checks: failing }] : [];
}),
bannerShown: tasks.flatMap(task => task.record.stagingMix?.retryBanner ? [{ taskId: task.taskId, firstShownAt: new Date(task.record.stagingMix.retryBanner.firstShownAt).toISOString(),
shownMs: bannerShownMs(task.record.stagingMix.retryBanner, now), errorCode: task.record.stagingMix.retryBanner.errorCode,
completedAnyway: task.finalStatus === "done" && !clicked(task) }] : []),
retryClicked: clickedTasks.length,
retryWorked: clickedTasks.filter(task => task.finalStatus === "done").length,
retryWorked: clickedTasks.filter(completed).length,
invariant,
};
}
@@ -113,6 +157,7 @@ export function summarizeStagingMix(tasks: readonly ResultTask[], extraClicks: R
export function stagingMixHeadline(result: StagingMixResult): string[] {
return [
`completed hands-off / started: ${result.completedHandsOff} / ${result.started}`,
`duplicated side effects: ${result.duplicatedSideEffects.length} tasks`,
`Retry banner shown: ${result.bannerShown.length} tasks`,
`Retry worked / Retry clicked: ${result.retryWorked} / ${result.retryClicked}`,
];
@@ -130,15 +175,15 @@ export async function readAssignedWorkInvariant(api: Pick<ChaosApi, "get">, comp
}
export async function writeStagingMixResults(input: { waveDir: string; api: Pick<ChaosApi, "get">; companyId: string;
tasks: ReadonlyArray<{ plan: TaskPlan; record: TaskRecord }>; passed: ReadonlyMap<string, boolean>;
timeline: ReadonlyArray<{ kind: string; [key: string]: unknown }> }): Promise<StagingMixResult> {
const extraClicks = new Map<string, boolean | null>(input.timeline.filter(entry => entry.kind === "release_swap_retry_clicked" && typeof entry.issueId === "string")
.map(entry => [String(entry.issueId), entry.accepted === true ? true : entry.accepted === false ? false : null]));
tasks: ReadonlyArray<{ plan: TaskPlan; record: TaskRecord }>; timeline: ReadonlyArray<{ kind: string; [key: string]: unknown }> }): Promise<StagingMixResult> {
const extraClicks = new Set(input.timeline.filter(entry => entry.kind === "release_swap_retry_clicked" && typeof entry.issueId === "string").map(entry => String(entry.issueId)));
const tasks: ResultTask[] = [];
for (const task of input.tasks) {
const issue = task.record.issueId ? await input.api.get<Json>(`/api/issues/${task.record.issueId}`, 10_000).catch(() => null) : null;
tasks.push({ taskId: task.plan.taskId, issueId: task.record.issueId, record: task.record, pass: input.passed.get(task.plan.taskId) === true,
finalStatus: issue?.status ?? task.record.finalStatus });
const grade = await readFile(path.join(input.waveDir, `${task.plan.taskId}.grade.json`), "utf8")
.then(text => JSON.parse(text) as { grade?: { T?: OracleResult } }).catch(() => null);
tasks.push({ taskId: task.plan.taskId, issueId: task.record.issueId, record: task.record, finalStatus: issue?.status ?? task.record.finalStatus,
checks: grade?.grade?.T?.checks ?? null });
}
const result = summarizeStagingMix(tasks, extraClicks, await readAssignedWorkInvariant(input.api, input.companyId));
await writeFile(path.join(input.waveDir, "staging-mix-results.json"), `${JSON.stringify({ schema: "paperclip.restart-chaos.staging-mix/v1",
@@ -146,6 +191,7 @@ export async function writeStagingMixResults(input: { waveDir: string; api: Pick
const invariant = result.invariant.violationCount === null ? `unreadable${result.invariant.error ? ` (${result.invariant.error})` : ""}`
: `${result.invariant.violationCount} violations, ${result.invariant.agentsWithStalledSlots} agents with stalled slots, ${result.invariant.openTasks} open tasks`;
await writeFile(path.join(input.waveDir, "staging-mix-summary.md"), [...stagingMixHeadline(result), `Assigned-work invariant: ${invariant}`, "",
...result.duplicatedSideEffects.map(entry => `- ${entry.taskId}: duplicated ${entry.checks.join(", ")}`),
...result.bannerShown.map(entry => `- ${entry.taskId}: Retry banner from ${entry.firstShownAt} for ${Math.round(entry.shownMs / 1000)} s`
+ `${entry.errorCode ? ` (${entry.errorCode})` : ""}${entry.completedAnyway ? ", completed without a click" : ""}`), ""].join("\n"), { mode: 0o600 });
return result;
+1 -1
View File
@@ -145,7 +145,7 @@ export class StagingMixRun {
async retryAfterSettle(observers: readonly RetryStageObserver[], api: Pick<ChaosApi, "get">, cancelled: () => boolean): Promise<void> {
if (this.mix.retryDuringWave) return;
await retryAfterSettleStage({ observers, api, settleUntil: this.settleUntil(), settleMs: this.mix.settleMinutes * 60_000, cancelled, note: this.note });
await retryAfterSettleStage({ observers, api, settleUntil: this.settleUntil(), cancelled, note: this.note });
}
mayLaunch(now: number, next: TaskPlan): boolean {
+19 -10
View File
@@ -63,8 +63,11 @@ export interface StagingMixArtifacts {
followUpErrors: string[];
retries: NonNullable<StagingMixRecord["retries"]>;
comments?: Array<{ body: string; runId: string | null }>;
commentsUnreadable?: boolean;
}
export const DUPLICATE_CHECKS: ReadonlySet<string> = new Set(["rounds_once", "done_comment_once", "completion_once", "subtasks_once", "subtask_rounds_once"]);
export const STAGING_MIX_ARG_OPTIONS = {
workload: { type: "string" },
"mix-runs": { type: "string" },
@@ -412,10 +415,12 @@ export function gradeStagingMix(nonce: string, task: StagingMixTask, artifacts:
const checks: OracleResult["checks"] = [
{ name: "issue_done", ok: artifacts.issueStatus === "done", detail: `status=${artifacts.issueStatus ?? "unknown"}` },
{ name: "ledger_rounds_verified", ok: own.verified, detail: own.detail },
{ name: "done_comment_posted", ok: doneComments >= 1, detail: `comments=${doneComments}` },
{ name: "rounds_once", ok: own.duplicates === 0, detail: `duplicates=${own.duplicates}` },
{ name: "done_comment_once", ok: doneComments === 1, detail: `comments=${doneComments}` },
{ name: "completion_once", ok: completionRuns.length <= 1,
detail: completionRuns.length > 1 ? `duplicate completion: runs ${completionRuns.map(id => id.slice(0, 8)).join(", ")} each reported completion` : `runs=${completionRuns.length}` },
{ name: "done_comment_once", ok: doneComments <= 1, detail: `comments=${doneComments}` },
artifacts.stagingMix?.commentsUnreadable ? { name: "completion_once", ok: false, detail: "unreadable: the task's comments could not be read" }
: { name: "completion_once", ok: completionRuns.length <= 1,
detail: completionRuns.length > 1 ? `duplicate completion: runs ${completionRuns.map(id => id.slice(0, 8)).join(", ")} each reported completion` : `runs=${completionRuns.length}` },
];
if (task.followUpAfterMs !== null) {
const errors = artifacts.stagingMix?.followUpErrors ?? [];
@@ -427,16 +432,20 @@ export function gradeStagingMix(nonce: string, task: StagingMixTask, artifacts:
detail: `posted ${new Date(followUp.at).toISOString()}${followUp.inOverlap ? " in overlap" : ""}` });
}
if (task.subtasks > 0) {
checks.push({ name: "subtasks_once", ok: children.length === task.subtasks, detail: `subtasks=${children.length}/${task.subtasks}` });
checks.push({ name: "subtasks_once", ok: children.length <= task.subtasks, detail: `subtasks=${children.length}/${task.subtasks}` });
let childDuplicates = 0;
for (let part = 1; part <= task.subtasks; part += 1) {
const nonceOf = childNonce(nonce, part);
const child = children.filter(candidate => (candidate.title ?? "").includes(nonceOf));
const result = verifyLedger(nonceOf, SUBTASK_ROUNDS, child[0]?.documents.ledger);
const results = child.map(candidate => ({ candidate, result: verifyLedger(nonceOf, SUBTASK_ROUNDS, candidate.documents.ledger) }));
const best = results.find(entry => entry.candidate.status === "done" && entry.result.verified) ?? results[0];
childDuplicates += results.reduce((sum, entry) => sum + entry.result.duplicates, 0);
const lastHash = expectedRoundHash(nonceOf, SUBTASK_ROUNDS);
checks.push({ name: `subtask_${part}_done`, ok: child.length === 1 && child[0]!.status === "done" && result.verified && result.duplicates === 0,
detail: child.length ? `status=${child[0]!.status} ${result.detail}` : "missing" });
checks.push({ name: `subtask_${part}_done`, ok: Boolean(best && best.candidate.status === "done" && best.result.verified),
detail: best ? `status=${best.candidate.status} ${best.result.detail}` : "missing" });
checks.push({ name: `parent_read_subtask_${part}`, ok: (ledger ?? "").includes(childLine(part, lastHash)), detail: childLine(part, lastHash) });
}
checks.push({ name: "subtask_rounds_once", ok: childDuplicates === 0, detail: `duplicates=${childDuplicates}` });
}
return { verdict: checks.every(check => check.ok) ? "pass" : "fail", checks };
}
@@ -561,8 +570,8 @@ export function bannerShownMs(banner: RetryBanner, now: number): number {
return banner.shownMs + (banner.visibleSince === null ? 0 : now - banner.visibleSince);
}
export async function collectStagingMixArtifacts(api: Pick<ChaosApi, "get">, issueId: string, record: TaskRecord): Promise<StagingMixArtifacts> {
const rows = await api.get<Array<{ body?: string; authorAgentId?: string | null; createdByRunId?: string | null }>>(`/api/issues/${issueId}/comments`).catch(() => null);
export function collectStagingMixArtifacts(record: TaskRecord, rows: ReadonlyArray<{ body?: unknown; authorAgentId?: unknown; createdByRunId?: unknown }> | null): StagingMixArtifacts {
return { followUp: record.stagingMix?.followUp ?? null, followUpErrors: record.stagingMix?.followUpErrors ?? [], retries: record.stagingMix?.retries ?? [],
...(rows ? { comments: rows.filter(row => row.authorAgentId).map(row => ({ body: String(row.body ?? ""), runId: row.createdByRunId ?? null })) } : {}) };
...(rows ? { comments: rows.filter(row => row.authorAgentId).map(row => ({ body: String(row.body ?? ""), runId: typeof row.createdByRunId === "string" ? row.createdByRunId : null })) }
: { commentsUnreadable: true }) };
}