mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 11:59:26 +01:00
test(restart-chaos): keep unreadable staging-mix evidence apart from confirmed duplicates
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
59b1b37870
commit
9907c4cdcc
3 files changed
+25
-3
No files matched your search
@@ -1413,7 +1413,7 @@ clicked during the wave, held until an overlap window as follow-ups are (`stagin
|
|||||||
**Results.** `staging-mix-results.json` and the first lines of `SUMMARY.md` give four lines, in this order:
|
**Results.** `staging-mix-results.json` and the first lines of `SUMMARY.md` give four lines, in this order:
|
||||||
|
|
||||||
1. `completed hands-off / started`.
|
1. `completed hands-off / started`.
|
||||||
2. `duplicated side effects: N tasks`.
|
2. `duplicated side effects: N tasks`, followed by `; evidence incomplete: M tasks` when a check could not read its evidence (such a check is never counted as a duplicate).
|
||||||
3. `Retry banner shown: N tasks`.
|
3. `Retry banner shown: N tasks`.
|
||||||
4. `Retry worked / Retry clicked`.
|
4. `Retry worked / Retry clicked`.
|
||||||
|
|
||||||
|
|||||||
@@ -130,10 +130,24 @@ describe("staging-mix Retry policy", () => {
|
|||||||
"Retry worked / Retry clicked: 1 / 2",
|
"Retry worked / Retry clicked: 1 / 2",
|
||||||
]);
|
]);
|
||||||
expect(result.duplicatedSideEffects).toEqual([{ taskId: "t2", checks: ["completion_once"] }]);
|
expect(result.duplicatedSideEffects).toEqual([{ taskId: "t2", checks: ["completion_once"] }]);
|
||||||
|
expect(result.evidenceIncomplete).toEqual([]);
|
||||||
expect(result.bannerShown.find(entry => entry.taskId === "t2")).toMatchObject({ completedAnyway: true, errorCode: "controller_lease_lost", shownMs: 30_000 });
|
expect(result.bannerShown.find(entry => entry.taskId === "t2")).toMatchObject({ completedAnyway: true, errorCode: "controller_lease_lost", shownMs: 30_000 });
|
||||||
expect(result.invariant.violationCount).toBe(2);
|
expect(result.invariant.violationCount).toBe(2);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("keeps an unreadable completion check out of the confirmed duplicates and says the evidence is incomplete", () => {
|
||||||
|
const record = { issueId: null, retry: null } as TaskRecord;
|
||||||
|
const checks = [{ name: "issue_done", ok: true }, { name: "ledger_rounds_verified", ok: true }, { name: "done_comment_posted", ok: true },
|
||||||
|
{ name: "completion_once", ok: false, detail: "unreadable: the task's comments could not be read" }];
|
||||||
|
const result = summarizeStagingMix([{ taskId: "t1", issueId: "i1", record, finalStatus: "done", checks },
|
||||||
|
{ taskId: "t2", issueId: "i2", record, finalStatus: "done", checks: [...checks.slice(0, 3), { name: "completion_once", ok: false, detail: "duplicate completion: runs a, b each reported completion" }] }],
|
||||||
|
new Set(), { violationCount: 0, violations: {}, agentsWithStalledSlots: 0, openTasks: 0 });
|
||||||
|
expect(result.duplicatedSideEffects).toEqual([{ taskId: "t2", checks: ["completion_once"] }]);
|
||||||
|
expect(result.evidenceIncomplete).toEqual([{ taskId: "t1", checks: ["completion_once"] }]);
|
||||||
|
expect(result.completedHandsOff).toBe(2);
|
||||||
|
expect(stagingMixHeadline(result)[1]).toBe("duplicated side effects: 1 tasks; evidence incomplete: 1 tasks");
|
||||||
|
});
|
||||||
|
|
||||||
it("reads the assigned-work invariant once as counts, and records a failed read", async () => {
|
it("reads the assigned-work invariant once as counts, and records a failed read", async () => {
|
||||||
const report = { checked: 4, openTasks: 3, violations: { nothing_pending: 1 }, violationCount: 1, agentsWithStalledSlots: [{ agentId: "a" }] };
|
const report = { checked: 4, openTasks: 3, violations: { nothing_pending: 1 }, violationCount: 1, agentsWithStalledSlots: [{ agentId: "a" }] };
|
||||||
expect(await readAssignedWorkInvariant({ get: async <T>() => report as T }, "c1"))
|
expect(await readAssignedWorkInvariant({ get: async <T>() => report as T }, "c1"))
|
||||||
|
|||||||
@@ -124,12 +124,15 @@ export interface StagingMixResult {
|
|||||||
started: number;
|
started: number;
|
||||||
completedHandsOff: number;
|
completedHandsOff: number;
|
||||||
duplicatedSideEffects: Array<{ taskId: string; checks: string[] }>;
|
duplicatedSideEffects: Array<{ taskId: string; checks: string[] }>;
|
||||||
|
evidenceIncomplete: Array<{ taskId: string; checks: string[] }>;
|
||||||
bannerShown: Array<{ taskId: string; firstShownAt: string; shownMs: number; errorCode: string | null; completedAnyway: boolean }>;
|
bannerShown: Array<{ taskId: string; firstShownAt: string; shownMs: number; errorCode: string | null; completedAnyway: boolean }>;
|
||||||
retryClicked: number;
|
retryClicked: number;
|
||||||
retryWorked: number;
|
retryWorked: number;
|
||||||
invariant: { violationCount: number | null; violations: Record<string, number> | null; agentsWithStalledSlots: number | null; openTasks: number | null; error?: string };
|
invariant: { violationCount: number | null; violations: Record<string, number> | null; agentsWithStalledSlots: number | null; openTasks: number | null; error?: string };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const isUnreadable = (check: { ok: boolean; detail?: string }) => !check.ok && (check.detail ?? "").startsWith("unreadable");
|
||||||
|
|
||||||
export function deliverableVerified(checks: OracleResult["checks"] | null): boolean {
|
export function deliverableVerified(checks: OracleResult["checks"] | null): boolean {
|
||||||
return Boolean(checks?.length) && checks!.every(check => check.ok || DUPLICATE_CHECKS.has(check.name));
|
return Boolean(checks?.length) && checks!.every(check => check.ok || DUPLICATE_CHECKS.has(check.name));
|
||||||
}
|
}
|
||||||
@@ -142,9 +145,13 @@ export function summarizeStagingMix(tasks: readonly ResultTask[], extraClicks: R
|
|||||||
started: tasks.length,
|
started: tasks.length,
|
||||||
completedHandsOff: tasks.filter(task => completed(task) && !clicked(task)).length,
|
completedHandsOff: tasks.filter(task => completed(task) && !clicked(task)).length,
|
||||||
duplicatedSideEffects: tasks.flatMap(task => {
|
duplicatedSideEffects: tasks.flatMap(task => {
|
||||||
const failing = (task.checks ?? []).filter(check => !check.ok && DUPLICATE_CHECKS.has(check.name)).map(check => check.name);
|
const failing = (task.checks ?? []).filter(check => !check.ok && DUPLICATE_CHECKS.has(check.name) && !isUnreadable(check)).map(check => check.name);
|
||||||
return failing.length ? [{ taskId: task.taskId, checks: failing }] : [];
|
return failing.length ? [{ taskId: task.taskId, checks: failing }] : [];
|
||||||
}),
|
}),
|
||||||
|
evidenceIncomplete: tasks.flatMap(task => {
|
||||||
|
const unreadable = (task.checks ?? []).filter(check => !check.ok && isUnreadable(check)).map(check => check.name);
|
||||||
|
return unreadable.length ? [{ taskId: task.taskId, checks: unreadable }] : [];
|
||||||
|
}),
|
||||||
bannerShown: tasks.flatMap(task => task.record.stagingMix?.retryBanner ? [{ taskId: task.taskId, firstShownAt: new Date(task.record.stagingMix.retryBanner.firstShownAt).toISOString(),
|
bannerShown: tasks.flatMap(task => task.record.stagingMix?.retryBanner ? [{ taskId: task.taskId, firstShownAt: new Date(task.record.stagingMix.retryBanner.firstShownAt).toISOString(),
|
||||||
shownMs: bannerShownMs(task.record.stagingMix.retryBanner, now), errorCode: task.record.stagingMix.retryBanner.errorCode,
|
shownMs: bannerShownMs(task.record.stagingMix.retryBanner, now), errorCode: task.record.stagingMix.retryBanner.errorCode,
|
||||||
completedAnyway: task.finalStatus === "done" && !clicked(task) }] : []),
|
completedAnyway: task.finalStatus === "done" && !clicked(task) }] : []),
|
||||||
@@ -157,7 +164,7 @@ export function summarizeStagingMix(tasks: readonly ResultTask[], extraClicks: R
|
|||||||
export function stagingMixHeadline(result: StagingMixResult): string[] {
|
export function stagingMixHeadline(result: StagingMixResult): string[] {
|
||||||
return [
|
return [
|
||||||
`completed hands-off / started: ${result.completedHandsOff} / ${result.started}`,
|
`completed hands-off / started: ${result.completedHandsOff} / ${result.started}`,
|
||||||
`duplicated side effects: ${result.duplicatedSideEffects.length} tasks`,
|
`duplicated side effects: ${result.duplicatedSideEffects.length} tasks${result.evidenceIncomplete.length ? `; evidence incomplete: ${result.evidenceIncomplete.length} tasks` : ""}`,
|
||||||
`Retry banner shown: ${result.bannerShown.length} tasks`,
|
`Retry banner shown: ${result.bannerShown.length} tasks`,
|
||||||
`Retry worked / Retry clicked: ${result.retryWorked} / ${result.retryClicked}`,
|
`Retry worked / Retry clicked: ${result.retryWorked} / ${result.retryClicked}`,
|
||||||
];
|
];
|
||||||
@@ -192,6 +199,7 @@ export async function writeStagingMixResults(input: { waveDir: string; api: Pick
|
|||||||
: `${result.invariant.violationCount} violations, ${result.invariant.agentsWithStalledSlots} agents with stalled slots, ${result.invariant.openTasks} open tasks`;
|
: `${result.invariant.violationCount} violations, ${result.invariant.agentsWithStalledSlots} agents with stalled slots, ${result.invariant.openTasks} open tasks`;
|
||||||
await writeFile(path.join(input.waveDir, "staging-mix-summary.md"), [...stagingMixHeadline(result), `Assigned-work invariant: ${invariant}`, "",
|
await writeFile(path.join(input.waveDir, "staging-mix-summary.md"), [...stagingMixHeadline(result), `Assigned-work invariant: ${invariant}`, "",
|
||||||
...result.duplicatedSideEffects.map(entry => `- ${entry.taskId}: duplicated ${entry.checks.join(", ")}`),
|
...result.duplicatedSideEffects.map(entry => `- ${entry.taskId}: duplicated ${entry.checks.join(", ")}`),
|
||||||
|
...result.evidenceIncomplete.map(entry => `- ${entry.taskId}: evidence incomplete for ${entry.checks.join(", ")}`),
|
||||||
...result.bannerShown.map(entry => `- ${entry.taskId}: Retry banner from ${entry.firstShownAt} for ${Math.round(entry.shownMs / 1000)} s`
|
...result.bannerShown.map(entry => `- ${entry.taskId}: Retry banner from ${entry.firstShownAt} for ${Math.round(entry.shownMs / 1000)} s`
|
||||||
+ `${entry.errorCode ? ` (${entry.errorCode})` : ""}${entry.completedAnyway ? ", completed without a click" : ""}`), ""].join("\n"), { mode: 0o600 });
|
+ `${entry.errorCode ? ` (${entry.errorCode})` : ""}${entry.completedAnyway ? ", completed without a click" : ""}`), ""].join("\n"), { mode: 0o600 });
|
||||||
return result;
|
return result;
|
||||||
|
|||||||
Reference in new issue
Block a user