test(restart-chaos): keep unreadable staging-mix evidence apart from confirmed duplicates

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-12 13:21:08 -06:00
1 parent 59b1b37870
commit 9907c4cdcc
3 files changed
+25 -3

No files matched your search

+1 -1
View File
@@ -1413,7 +1413,7 @@ clicked during the wave, held until an overlap window as follow-ups are (`stagin
**Results.** `staging-mix-results.json` and the first lines of `SUMMARY.md` give four lines, in this order:
1. `completed hands-off / started`.
2. `duplicated side effects: N tasks`.
2. `duplicated side effects: N tasks`, followed by `; evidence incomplete: M tasks` when a check could not read its evidence (such a check is never counted as a duplicate).
3. `Retry banner shown: N tasks`.
4. `Retry worked / Retry clicked`.
@@ -130,10 +130,24 @@ describe("staging-mix Retry policy", () => {
"Retry worked / Retry clicked: 1 / 2",
]);
expect(result.duplicatedSideEffects).toEqual([{ taskId: "t2", checks: ["completion_once"] }]);
expect(result.evidenceIncomplete).toEqual([]);
expect(result.bannerShown.find(entry => entry.taskId === "t2")).toMatchObject({ completedAnyway: true, errorCode: "controller_lease_lost", shownMs: 30_000 });
expect(result.invariant.violationCount).toBe(2);
});
it("keeps an unreadable completion check out of the confirmed duplicates and says the evidence is incomplete", () => {
const record = { issueId: null, retry: null } as TaskRecord;
const checks = [{ name: "issue_done", ok: true }, { name: "ledger_rounds_verified", ok: true }, { name: "done_comment_posted", ok: true },
{ name: "completion_once", ok: false, detail: "unreadable: the task's comments could not be read" }];
const result = summarizeStagingMix([{ taskId: "t1", issueId: "i1", record, finalStatus: "done", checks },
{ taskId: "t2", issueId: "i2", record, finalStatus: "done", checks: [...checks.slice(0, 3), { name: "completion_once", ok: false, detail: "duplicate completion: runs a, b each reported completion" }] }],
new Set(), { violationCount: 0, violations: {}, agentsWithStalledSlots: 0, openTasks: 0 });
expect(result.duplicatedSideEffects).toEqual([{ taskId: "t2", checks: ["completion_once"] }]);
expect(result.evidenceIncomplete).toEqual([{ taskId: "t1", checks: ["completion_once"] }]);
expect(result.completedHandsOff).toBe(2);
expect(stagingMixHeadline(result)[1]).toBe("duplicated side effects: 1 tasks; evidence incomplete: 1 tasks");
});
it("reads the assigned-work invariant once as counts, and records a failed read", async () => {
const report = { checked: 4, openTasks: 3, violations: { nothing_pending: 1 }, violationCount: 1, agentsWithStalledSlots: [{ agentId: "a" }] };
expect(await readAssignedWorkInvariant({ get: async <T>() => report as T }, "c1"))
+10 -2
View File
@@ -124,12 +124,15 @@ export interface StagingMixResult {
started: number;
completedHandsOff: number;
duplicatedSideEffects: Array<{ taskId: string; checks: string[] }>;
evidenceIncomplete: Array<{ taskId: string; checks: string[] }>;
bannerShown: Array<{ taskId: string; firstShownAt: string; shownMs: number; errorCode: string | null; completedAnyway: boolean }>;
retryClicked: number;
retryWorked: number;
invariant: { violationCount: number | null; violations: Record<string, number> | null; agentsWithStalledSlots: number | null; openTasks: number | null; error?: string };
}
export const isUnreadable = (check: { ok: boolean; detail?: string }) => !check.ok && (check.detail ?? "").startsWith("unreadable");
export function deliverableVerified(checks: OracleResult["checks"] | null): boolean {
return Boolean(checks?.length) && checks!.every(check => check.ok || DUPLICATE_CHECKS.has(check.name));
}
@@ -142,9 +145,13 @@ export function summarizeStagingMix(tasks: readonly ResultTask[], extraClicks: R
started: tasks.length,
completedHandsOff: tasks.filter(task => completed(task) && !clicked(task)).length,
duplicatedSideEffects: tasks.flatMap(task => {
const failing = (task.checks ?? []).filter(check => !check.ok && DUPLICATE_CHECKS.has(check.name)).map(check => check.name);
const failing = (task.checks ?? []).filter(check => !check.ok && DUPLICATE_CHECKS.has(check.name) && !isUnreadable(check)).map(check => check.name);
return failing.length ? [{ taskId: task.taskId, checks: failing }] : [];
}),
evidenceIncomplete: tasks.flatMap(task => {
const unreadable = (task.checks ?? []).filter(check => !check.ok && isUnreadable(check)).map(check => check.name);
return unreadable.length ? [{ taskId: task.taskId, checks: unreadable }] : [];
}),
bannerShown: tasks.flatMap(task => task.record.stagingMix?.retryBanner ? [{ taskId: task.taskId, firstShownAt: new Date(task.record.stagingMix.retryBanner.firstShownAt).toISOString(),
shownMs: bannerShownMs(task.record.stagingMix.retryBanner, now), errorCode: task.record.stagingMix.retryBanner.errorCode,
completedAnyway: task.finalStatus === "done" && !clicked(task) }] : []),
@@ -157,7 +164,7 @@ export function summarizeStagingMix(tasks: readonly ResultTask[], extraClicks: R
export function stagingMixHeadline(result: StagingMixResult): string[] {
return [
`completed hands-off / started: ${result.completedHandsOff} / ${result.started}`,
`duplicated side effects: ${result.duplicatedSideEffects.length} tasks`,
`duplicated side effects: ${result.duplicatedSideEffects.length} tasks${result.evidenceIncomplete.length ? `; evidence incomplete: ${result.evidenceIncomplete.length} tasks` : ""}`,
`Retry banner shown: ${result.bannerShown.length} tasks`,
`Retry worked / Retry clicked: ${result.retryWorked} / ${result.retryClicked}`,
];
@@ -192,6 +199,7 @@ export async function writeStagingMixResults(input: { waveDir: string; api: Pick
: `${result.invariant.violationCount} violations, ${result.invariant.agentsWithStalledSlots} agents with stalled slots, ${result.invariant.openTasks} open tasks`;
await writeFile(path.join(input.waveDir, "staging-mix-summary.md"), [...stagingMixHeadline(result), `Assigned-work invariant: ${invariant}`, "",
...result.duplicatedSideEffects.map(entry => `- ${entry.taskId}: duplicated ${entry.checks.join(", ")}`),
...result.evidenceIncomplete.map(entry => `- ${entry.taskId}: evidence incomplete for ${entry.checks.join(", ")}`),
...result.bannerShown.map(entry => `- ${entry.taskId}: Retry banner from ${entry.firstShownAt} for ${Math.round(entry.shownMs / 1000)} s`
+ `${entry.errorCode ? ` (${entry.errorCode})` : ""}${entry.completedAnyway ? ", completed without a click" : ""}`), ""].join("\n"), { mode: 0o600 });
return result;