From 9907c4cdccc8c2608f4e330a29b1fb1e688e6ae2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 12 Nov 2026 13:21:08 -0600 Subject: [PATCH] test(restart-chaos): keep unreadable staging-mix evidence apart from confirmed duplicates Co-Authored-By: Claude Opus 5.5 --- tests/restart-chaos/README.md | 2 +- tests/restart-chaos/staging-mix-retry.test.ts | 14 ++++++++++++++ tests/restart-chaos/staging-mix-retry.ts | 12 ++++++++++-- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index c87edfbdab2..d2dd47cd66f 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -1413,7 +1413,7 @@ clicked during the wave, held until an overlap window as follow-ups are (`stagin **Results.** `staging-mix-results.json` and the first lines of `SUMMARY.md` give four lines, in this order: 1. `completed hands-off / started`. -2. `duplicated side effects: N tasks`. +2. `duplicated side effects: N tasks`, followed by `; evidence incomplete: M tasks` when a check could not read its evidence (such a check is never counted as a duplicate). 3. `Retry banner shown: N tasks`. 4. `Retry worked / Retry clicked`. diff --git a/tests/restart-chaos/staging-mix-retry.test.ts b/tests/restart-chaos/staging-mix-retry.test.ts index 59db481fb51..3d52bfd1a09 100644 --- a/tests/restart-chaos/staging-mix-retry.test.ts +++ b/tests/restart-chaos/staging-mix-retry.test.ts @@ -130,10 +130,24 @@ describe("staging-mix Retry policy", () => { "Retry worked / Retry clicked: 1 / 2", ]); expect(result.duplicatedSideEffects).toEqual([{ taskId: "t2", checks: ["completion_once"] }]); + expect(result.evidenceIncomplete).toEqual([]); expect(result.bannerShown.find(entry => entry.taskId === "t2")).toMatchObject({ completedAnyway: true, errorCode: "controller_lease_lost", shownMs: 30_000 }); expect(result.invariant.violationCount).toBe(2); }); + it("keeps an unreadable completion check out of the confirmed duplicates and says the evidence is incomplete", () => { + const record = { issueId: null, retry: null } as TaskRecord; + const checks = [{ name: "issue_done", ok: true }, { name: "ledger_rounds_verified", ok: true }, { name: "done_comment_posted", ok: true }, + { name: "completion_once", ok: false, detail: "unreadable: the task's comments could not be read" }]; + const result = summarizeStagingMix([{ taskId: "t1", issueId: "i1", record, finalStatus: "done", checks }, + { taskId: "t2", issueId: "i2", record, finalStatus: "done", checks: [...checks.slice(0, 3), { name: "completion_once", ok: false, detail: "duplicate completion: runs a, b each reported completion" }] }], + new Set(), { violationCount: 0, violations: {}, agentsWithStalledSlots: 0, openTasks: 0 }); + expect(result.duplicatedSideEffects).toEqual([{ taskId: "t2", checks: ["completion_once"] }]); + expect(result.evidenceIncomplete).toEqual([{ taskId: "t1", checks: ["completion_once"] }]); + expect(result.completedHandsOff).toBe(2); + expect(stagingMixHeadline(result)[1]).toBe("duplicated side effects: 1 tasks; evidence incomplete: 1 tasks"); + }); + it("reads the assigned-work invariant once as counts, and records a failed read", async () => { const report = { checked: 4, openTasks: 3, violations: { nothing_pending: 1 }, violationCount: 1, agentsWithStalledSlots: [{ agentId: "a" }] }; expect(await readAssignedWorkInvariant({ get: async () => report as T }, "c1")) diff --git a/tests/restart-chaos/staging-mix-retry.ts b/tests/restart-chaos/staging-mix-retry.ts index 9b753c28b5e..78d51694ee5 100644 --- a/tests/restart-chaos/staging-mix-retry.ts +++ b/tests/restart-chaos/staging-mix-retry.ts @@ -124,12 +124,15 @@ export interface StagingMixResult { started: number; completedHandsOff: number; duplicatedSideEffects: Array<{ taskId: string; checks: string[] }>; + evidenceIncomplete: Array<{ taskId: string; checks: string[] }>; bannerShown: Array<{ taskId: string; firstShownAt: string; shownMs: number; errorCode: string | null; completedAnyway: boolean }>; retryClicked: number; retryWorked: number; invariant: { violationCount: number | null; violations: Record | null; agentsWithStalledSlots: number | null; openTasks: number | null; error?: string }; } +export const isUnreadable = (check: { ok: boolean; detail?: string }) => !check.ok && (check.detail ?? "").startsWith("unreadable"); + export function deliverableVerified(checks: OracleResult["checks"] | null): boolean { return Boolean(checks?.length) && checks!.every(check => check.ok || DUPLICATE_CHECKS.has(check.name)); } @@ -142,9 +145,13 @@ export function summarizeStagingMix(tasks: readonly ResultTask[], extraClicks: R started: tasks.length, completedHandsOff: tasks.filter(task => completed(task) && !clicked(task)).length, duplicatedSideEffects: tasks.flatMap(task => { - const failing = (task.checks ?? []).filter(check => !check.ok && DUPLICATE_CHECKS.has(check.name)).map(check => check.name); + const failing = (task.checks ?? []).filter(check => !check.ok && DUPLICATE_CHECKS.has(check.name) && !isUnreadable(check)).map(check => check.name); return failing.length ? [{ taskId: task.taskId, checks: failing }] : []; }), + evidenceIncomplete: tasks.flatMap(task => { + const unreadable = (task.checks ?? []).filter(check => !check.ok && isUnreadable(check)).map(check => check.name); + return unreadable.length ? [{ taskId: task.taskId, checks: unreadable }] : []; + }), bannerShown: tasks.flatMap(task => task.record.stagingMix?.retryBanner ? [{ taskId: task.taskId, firstShownAt: new Date(task.record.stagingMix.retryBanner.firstShownAt).toISOString(), shownMs: bannerShownMs(task.record.stagingMix.retryBanner, now), errorCode: task.record.stagingMix.retryBanner.errorCode, completedAnyway: task.finalStatus === "done" && !clicked(task) }] : []), @@ -157,7 +164,7 @@ export function summarizeStagingMix(tasks: readonly ResultTask[], extraClicks: R export function stagingMixHeadline(result: StagingMixResult): string[] { return [ `completed hands-off / started: ${result.completedHandsOff} / ${result.started}`, - `duplicated side effects: ${result.duplicatedSideEffects.length} tasks`, + `duplicated side effects: ${result.duplicatedSideEffects.length} tasks${result.evidenceIncomplete.length ? `; evidence incomplete: ${result.evidenceIncomplete.length} tasks` : ""}`, `Retry banner shown: ${result.bannerShown.length} tasks`, `Retry worked / Retry clicked: ${result.retryWorked} / ${result.retryClicked}`, ]; @@ -192,6 +199,7 @@ export async function writeStagingMixResults(input: { waveDir: string; api: Pick : `${result.invariant.violationCount} violations, ${result.invariant.agentsWithStalledSlots} agents with stalled slots, ${result.invariant.openTasks} open tasks`; await writeFile(path.join(input.waveDir, "staging-mix-summary.md"), [...stagingMixHeadline(result), `Assigned-work invariant: ${invariant}`, "", ...result.duplicatedSideEffects.map(entry => `- ${entry.taskId}: duplicated ${entry.checks.join(", ")}`), + ...result.evidenceIncomplete.map(entry => `- ${entry.taskId}: evidence incomplete for ${entry.checks.join(", ")}`), ...result.bannerShown.map(entry => `- ${entry.taskId}: Retry banner from ${entry.firstShownAt} for ${Math.round(entry.shownMs / 1000)} s` + `${entry.errorCode ? ` (${entry.errorCode})` : ""}${entry.completedAnyway ? ", completed without a click" : ""}`), ""].join("\n"), { mode: 0o600 }); return result;