mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 11:59:26 +01:00
test(restart-chaos): count hands-off completion from the deliverable and never click a scheduled retry
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ddb64d3223
commit
59b1b37870
9 files changed
+228
-92
No files matched your search
@@ -1397,15 +1397,35 @@ own continuation is what gets measured. A task that shows a Retry banner is kept
|
||||
is recorded per task: when it first showed, how long it stayed, and the error code behind it, even when the task then
|
||||
finished by itself. A banner is a defect, because the product goal is that it never appears. After the settle
|
||||
(`--mix-settle-minutes`, default 15, after the last fault slot ends), each task that is still not done and still
|
||||
offers Retry gets one click (`staging_mix_retry_after_settle`), and the harness follows it to rest. When the release-swap
|
||||
cell drives the wave, its own end-of-wave pass makes that one click instead. Both paths use
|
||||
`shouldClickRetryAfterSettle` in `staging-mix-retry.ts`: never on a done task, a task waiting on a scheduled retry, or a
|
||||
task that was already clicked. So no task gets two clicks. `--mix-retry-during-wave` restores the old behaviour: Retries
|
||||
offers Retry gets one click (`staging_mix_retry_after_settle`). The harness follows the click to rest for up to 15 minutes,
|
||||
whatever the settle setting. A task that finished by itself during the settle is watched again until it rests, so its
|
||||
record is fresh before grading. When the release-swap cell drives the wave, its own end-of-wave pass makes that one
|
||||
click instead, and its issue reads also update the banner record. Both paths use `shouldClickRetryAfterSettle` in
|
||||
`staging-mix-retry.ts`, which never clicks:
|
||||
|
||||
- a done task;
|
||||
- a task with a scheduled retry (`scheduledRetry`, state `retry_scheduled` or a `nextRetryAt`);
|
||||
- a task that was already clicked.
|
||||
|
||||
So no task gets two clicks. `--mix-retry-during-wave` restores the old behaviour: Retries
|
||||
clicked during the wave, held until an overlap window as follow-ups are (`staging_mix_retry_in_overlap`).
|
||||
|
||||
**Results.** `staging-mix-results.json` and the first lines of `SUMMARY.md` give three numbers, in this order:
|
||||
`completed hands-off / started`, `Retry banner shown: N tasks` and `Retry worked / Retry clicked`. A Retry counts as
|
||||
worked when the task is done at the end. The banner list follows, then one read of
|
||||
**Results.** `staging-mix-results.json` and the first lines of `SUMMARY.md` give four lines, in this order:
|
||||
|
||||
1. `completed hands-off / started`.
|
||||
2. `duplicated side effects: N tasks`.
|
||||
3. `Retry banner shown: N tasks`.
|
||||
4. `Retry worked / Retry clicked`.
|
||||
|
||||
A task counts as completed hands-off when all three hold, read from fresh data at the end of the wave:
|
||||
|
||||
- its status is `done`;
|
||||
- every check of its deliverable passes (ledger rounds, DONE comment posted, follow-up delivered, subtasks);
|
||||
- nobody clicked Retry.
|
||||
|
||||
A duplicated side effect (`rounds_once`, `done_comment_once`, `completion_once`, `subtasks_once`,
|
||||
`subtask_rounds_once`) is counted on the second line. It does not remove the task from the first line. A Retry counts as
|
||||
worked when the clicked task meets the same completion rule. The banner list follows, then one read of
|
||||
`GET /api/companies/:id/assigned-work-invariant` at the end of the wave: `violationCount`, `violations` by reason,
|
||||
`agentsWithStalledSlots` and `openTasks`.
|
||||
|
||||
@@ -1417,11 +1437,13 @@ not the status:
|
||||
| `issue_done` | the task is not `done` |
|
||||
| `ledger_rounds_verified` | a round is missing, ran under 80 s, or its hash differs from the harness's own computation (POSIX `cksum`, chained per tick) |
|
||||
| `rounds_once` | a round line appears twice (a tool ran again) |
|
||||
| `done_comment_once` | the comment whose whole body is `DONE-<nonce>` is missing or was posted twice; a summary that quotes the marker does not count |
|
||||
| `completion_once` | more than one run posted a comment that mentions `DONE-<nonce>` (a duplicate completion) |
|
||||
| `done_comment_posted` | no comment's whole body is `DONE-<nonce>`; a summary that quotes the marker does not count |
|
||||
| `done_comment_once` | that comment was posted more than once |
|
||||
| `completion_once` | more than one run posted a comment that mentions `DONE-<nonce>` (a duplicate completion), or the comments could not be read |
|
||||
| `follow_up_posted` | a planned follow-up was never posted (the detail lists the post errors) |
|
||||
| `follow_up_served` | a follow-up was posted during a run and the ledger lacks its line |
|
||||
| `subtasks_once`, `subtask_<k>_done`, `parent_read_subtask_<k>` | a parent created more or fewer than two subtasks, a subtask's ledger fails, or the parent did not report a subtask's last hash |
|
||||
| `subtask_<k>_done`, `parent_read_subtask_<k>` | a subtask is missing, not done or its ledger fails, or the parent did not report its last hash |
|
||||
| `subtasks_once`, `subtask_rounds_once` | a parent created more than two subtasks, or a subtask repeated a round |
|
||||
|
||||
Duplicated side effects (a repeated round, comment or subtask) fail the task and show in the signature.
|
||||
|
||||
|
||||
@@ -836,7 +836,7 @@ async function main(): Promise<number> {
|
||||
rules: Object.fromEntries(Object.entries(grade.rules).map(([id, rule]) => [id, rule.verdict])), T: grade.T.verdict,
|
||||
universal: Object.fromEntries(Object.entries(grade.universal ?? {}).map(([id, rule]) => [id, `${rule.verdict}${rule.enforced ? "" : " (warning)"}`])) });
|
||||
}
|
||||
if (mix) await writeStagingMixResults({ waveDir, api, companyId: lab.companyId, tasks: launched, passed: new Map(lines.map(line => [line.taskId, line.pass])), timeline: timeline.entries })
|
||||
if (mix) await writeStagingMixResults({ waveDir, api, companyId: lab.companyId, tasks: launched, timeline: timeline.entries })
|
||||
.catch(error => { harnessError ??= new Error(`Staging-mix results failed: ${String(error)}`); });
|
||||
// MS1–MS3: the cross-server invariant over the whole wave (multi-server.ts).
|
||||
if (faults.some(receipt => receipt.multiServer)) {
|
||||
|
||||
@@ -698,10 +698,15 @@ export class TaskObserver {
|
||||
canRetry: sample.interruption?.canRetry ?? false, retried: Boolean(this.record.retry) });
|
||||
}
|
||||
|
||||
async retryAfterSettle(sample: TaskSample, now: number): Promise<void> {
|
||||
reopen(now: number): void {
|
||||
this.done = false;
|
||||
this.record.restAt = null;
|
||||
this.record.timedOut = false;
|
||||
this.record.deadlineAt = now + RETRY_BUDGET_MS;
|
||||
}
|
||||
|
||||
async retryAfterSettle(sample: TaskSample, now: number): Promise<void> {
|
||||
this.reopen(now);
|
||||
await this.boardRetry(sample, now);
|
||||
this.record.deadlineAt = now + RETRY_BUDGET_MS;
|
||||
}
|
||||
@@ -1043,7 +1048,7 @@ export class TaskObserver {
|
||||
const document = await this.api.get<Json>(`/api/issues/${issueId}/documents/${key}`).catch(() => null);
|
||||
documents[key] = typeof document?.body === "string" ? document.body : null;
|
||||
}
|
||||
const comments = await this.api.get<Json[]>(`/api/issues/${issueId}/comments`).catch(() => []);
|
||||
const comments = await this.api.get<Json[]>(`/api/issues/${issueId}/comments`).catch(() => null);
|
||||
const workspacePath = await this.workspacePath();
|
||||
const snapshotPath = this.record.preFaultWorkspacePath ?? workspacePath;
|
||||
if (snapshotPath && this.record.preFaultFiles) {
|
||||
@@ -1101,7 +1106,7 @@ export class TaskObserver {
|
||||
status: String(run.status), errorCode: run.errorCode ?? null, createdAt: run.createdAt ?? null, finishedAt: run.finishedAt ?? null }))).catch(() => []),
|
||||
...(this.journey ? { journey: await this.journey.collect(issueId).finally(() => this.journey!.close()) } : {}),
|
||||
...(this.plan.workload === "H2" ? { handoff: await this.collectHandoff(issueId, issue) } : {}),
|
||||
...(this.plan.stagingMix ? { stagingMix: await collectStagingMixArtifacts(this.api, issueId, this.record) } : {}),
|
||||
...(this.plan.stagingMix ? { stagingMix: collectStagingMixArtifacts(this.record, comments) } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ export interface ReleaseSwapFinishInput {
|
||||
record: (kind: string, data: Record<string, unknown>) => void;
|
||||
pollMs?: number;
|
||||
drainCapMs?: number;
|
||||
onView?: (issueId: string, issue: Json | null, at: number) => void;
|
||||
}
|
||||
|
||||
type Client = Pick<Db, "$client">;
|
||||
@@ -200,6 +201,7 @@ export async function finishReleaseSwap(input: ReleaseSwapFinishInput): Promise<
|
||||
const left = deadline - Date.now();
|
||||
views.set(id, left <= 0 ? { status: "unreadable", error: "view deadline passed" }
|
||||
: await input.api.get<Json>(`/api/issues/${id}`, Math.min(10_000, left)).catch(error => ({ status: "unreadable", error: String(error) })));
|
||||
input.onView?.(id, views.get(id)!, Date.now());
|
||||
}
|
||||
}));
|
||||
};
|
||||
|
||||
@@ -7,6 +7,7 @@ import { ReleaseSwapRun, overlapLaunchExtensionMs, type ReleaseSwapOptions } fro
|
||||
import { finishReleaseSwap, type ReleaseSwapFinishInput } from "./release-swap-finish.js";
|
||||
import { StormRun } from "./scheduler.js";
|
||||
import { setOverlapProbe } from "./staging-mix.js";
|
||||
import { bannerSink } from "./staging-mix-retry.js";
|
||||
import type { ChaosSupervisor } from "./supervisor.js";
|
||||
import type { TaskObserver } from "./observer.js";
|
||||
import { waitForAgentsReady, type ChaosLab } from "./seed.js";
|
||||
@@ -87,7 +88,8 @@ export class ReleaseSwapWave {
|
||||
const graded = await finishReleaseSwap({ driver: this.driver, settleMs: options.settleMs, companyId: this.lab.companyId, api: this.api, db: this.db, board: input.board,
|
||||
drainCapMs: options.drainMs ?? Math.max(0, ...input.observers.map(observer => observer.definition.budgetMs)) + options.settleMs,
|
||||
timeline: this.timeline.entries, issueIdForTask: new Map(input.launched.flatMap(observer => observer.record.issueId ? [[observer.plan.taskId, observer.record.issueId] as const] : [])),
|
||||
serverWindows: this.supervisor.serverWindows, waveDir: input.waveDir, cancelled: this.cancelled, record: (kind, data) => { this.timeline.add(kind, data); } });
|
||||
serverWindows: this.supervisor.serverWindows, waveDir: input.waveDir, cancelled: this.cancelled, record: (kind, data) => { this.timeline.add(kind, data); },
|
||||
onView: bannerSink(input.launched) });
|
||||
return graded.pass;
|
||||
} finally {
|
||||
setOverlapProbe(null);
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseCommand } from "./config.js";
|
||||
import { doneMarker, expectedRoundHash, gradeStagingMix, planStagingMixWave, ROUND_SECONDS, type StagingMixTask } from "./staging-mix.js";
|
||||
import { collectStagingMixArtifacts, doneMarker, expectedRoundHash, gradeStagingMix, planStagingMixWave, ROUND_SECONDS, type StagingMixTask } from "./staging-mix.js";
|
||||
import {
|
||||
readAssignedWorkInvariant, retryAfterSettleStage, shouldClickRetryAfterSettle, stagingMixHeadline, summarizeStagingMix, type RetryStageObserver,
|
||||
bannerSink, issueView, readAssignedWorkInvariant, retryAfterSettleStage, shouldClickRetryAfterSettle, stagingMixHeadline, summarizeStagingMix,
|
||||
type RetryStageObserver,
|
||||
} from "./staging-mix-retry.js";
|
||||
import type { OracleArtifacts, TaskPlan, TaskRecord, TaskSample } from "./types.js";
|
||||
|
||||
@@ -30,63 +31,105 @@ describe("staging-mix Retry policy", () => {
|
||||
expect(planStagingMixWave("w", old).tasks.every(entry => entry.stagingMix!.retryDuringWave === true)).toBe(true);
|
||||
});
|
||||
|
||||
it("clicks once after the settle, only on a task still offering Retry that nobody clicked", () => {
|
||||
expect(shouldClickRetryAfterSettle({ status: "blocked", canRetry: true, retryWait: false }, 0)).toBe(true);
|
||||
expect(shouldClickRetryAfterSettle({ status: "blocked", canRetry: true, retryWait: false }, 1)).toBe(false);
|
||||
expect(shouldClickRetryAfterSettle({ status: "done", canRetry: true, retryWait: false }, 0)).toBe(false);
|
||||
expect(shouldClickRetryAfterSettle({ status: "blocked", canRetry: true, retryWait: true }, 0)).toBe(false);
|
||||
expect(shouldClickRetryAfterSettle({ status: "in_progress", canRetry: false, retryWait: false }, 0)).toBe(false);
|
||||
it("clicks once after the settle, only on a task still offering Retry that nobody clicked and that has no retry scheduled", () => {
|
||||
const view = (overrides: Partial<ReturnType<typeof issueView>> = {}) => ({ status: "blocked", canRetry: true, retryWait: false, retryScheduled: false, ...overrides });
|
||||
expect(shouldClickRetryAfterSettle(view(), 0)).toBe(true);
|
||||
expect(shouldClickRetryAfterSettle(view(), 1)).toBe(false);
|
||||
expect(shouldClickRetryAfterSettle(view({ status: "done" }), 0)).toBe(false);
|
||||
expect(shouldClickRetryAfterSettle(view({ retryWait: true }), 0)).toBe(false);
|
||||
expect(shouldClickRetryAfterSettle(view({ canRetry: false }), 0)).toBe(false);
|
||||
expect(shouldClickRetryAfterSettle(view({ retryScheduled: true }), 0)).toBe(false);
|
||||
const scheduled = issueView({ status: "in_progress", interruption: { state: "retry_scheduled", canRetry: true, nextRetryAt: "2026-11-12T20:00:00Z" } });
|
||||
expect(scheduled.retryScheduled).toBe(true);
|
||||
expect(shouldClickRetryAfterSettle(scheduled, 0)).toBe(false);
|
||||
expect(issueView({ status: "blocked", scheduledRetry: { scheduledRetryAt: "x" }, interruption: { state: "failed", canRetry: true } }).retryScheduled).toBe(true);
|
||||
expect(issueView({ status: "blocked", interruption: { state: "failed", canRetry: true } }).retryScheduled).toBe(false);
|
||||
});
|
||||
|
||||
it("waits out the settle hands-off, then clicks once per stuck task and follows it to rest", async () => {
|
||||
const harness = () => {
|
||||
let clock = 0;
|
||||
const issues: Record<string, Record<string, unknown>> = {
|
||||
i1: { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r1" } },
|
||||
i2: { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r2" } },
|
||||
i3: { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r3" } },
|
||||
};
|
||||
const observer = (taskId: string, issueId: string, retry: TaskRecord["retry"] = null): RetryStageObserver & { clicks: number } => {
|
||||
const issues: Record<string, Record<string, unknown>> = {};
|
||||
const observer = (taskId: string, issueId: string, overrides: Partial<TaskRecord> = {}, finishAfterClickMs = 0) => {
|
||||
const self = {
|
||||
plan: { taskId, stagingMix: task() } as TaskPlan, record: { issueId, retry } as TaskRecord, done: true, clicks: 0,
|
||||
async poll() { if (issues[issueId]!.status === "done") self.done = true; },
|
||||
plan: { taskId, stagingMix: task() } as TaskPlan, record: { issueId, retry: null, finalStatus: "blocked", ...overrides } as TaskRecord, done: true,
|
||||
clicks: 0, reopened: 0, clickedAt: null as number | null,
|
||||
async poll() {
|
||||
if (issues[issueId]!.status === "done" || (self.clickedAt !== null && clock - self.clickedAt >= finishAfterClickMs)) { issues[issueId] = { status: "done" }; self.done = true; }
|
||||
},
|
||||
reopen() { self.done = false; self.reopened += 1; },
|
||||
async retryAfterSettle(sample: TaskSample, at: number) {
|
||||
self.done = false; self.clicks += 1;
|
||||
self.done = false; self.clicks += 1; self.clickedAt = at;
|
||||
expect(sample.interruption?.observedRunId).toBe(`r${issueId.slice(1)}`);
|
||||
self.record.retry = { at, accepted: true } as TaskRecord["retry"];
|
||||
issues[issueId] = { status: "done" };
|
||||
},
|
||||
};
|
||||
return self;
|
||||
return self satisfies RetryStageObserver;
|
||||
};
|
||||
const healsItself = observer("t1", "i1"), stuck = observer("t2", "i2"), alreadyClicked = observer("t3", "i3", { at: 1, accepted: true } as TaskRecord["retry"]);
|
||||
return { issues, observer, now: () => clock, advance: (ms: number) => { clock += ms; } };
|
||||
};
|
||||
|
||||
it("waits out the settle hands-off, refreshes a task that healed, clicks once per stuck task and follows it on the retry budget", async () => {
|
||||
const h = harness();
|
||||
const failed = (id: string) => ({ status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: `r${id}` } });
|
||||
Object.assign(h.issues, { i1: failed("1"), i2: failed("2"), i3: failed("3"), i4: { status: "in_progress", interruption: { state: "retry_scheduled", canRetry: true } } });
|
||||
const healsItself = h.observer("t1", "i1"), stuck = h.observer("t2", "i2", {}, 10 * 60_000);
|
||||
const alreadyClicked = h.observer("t3", "i3", { retry: { at: 1, accepted: true } as TaskRecord["retry"] }), scheduled = h.observer("t4", "i4");
|
||||
const notes: string[] = [];
|
||||
const result = await retryAfterSettleStage({ observers: [healsItself, stuck, alreadyClicked], api: { get: async <T>(path: string) => issues[path.split("/").pop()!] as T },
|
||||
settleUntil: 120_000, settleMs: 60_000, cancelled: () => false, note: kind => notes.push(kind), pollMs: 60_000, now: () => clock,
|
||||
sleep: async ms => { clock += ms; if (clock >= 60_000) issues.i1 = { status: "done" }; } });
|
||||
expect(clock).toBeGreaterThanOrEqual(120_000);
|
||||
expect(result.clicked).toEqual(["t2"]);
|
||||
expect([healsItself.clicks, stuck.clicks, alreadyClicked.clicks]).toEqual([0, 1, 0]);
|
||||
expect(stuck.done).toBe(true);
|
||||
const result = await retryAfterSettleStage({ observers: [healsItself, stuck, alreadyClicked, scheduled],
|
||||
api: { get: async <T>(path: string) => h.issues[path.split("/").pop()!] as T }, settleUntil: 120_000, cancelled: () => false,
|
||||
note: kind => notes.push(kind), pollMs: 60_000, now: h.now, sleep: async ms => { h.advance(ms); if (h.now() >= 60_000) h.issues.i1 = { status: "done" }; } });
|
||||
expect(result).toEqual({ clicked: ["t2"], refreshed: ["t1"] });
|
||||
expect([healsItself.clicks, stuck.clicks, alreadyClicked.clicks, scheduled.clicks]).toEqual([0, 1, 0, 0]);
|
||||
expect(healsItself.reopened).toBe(1);
|
||||
expect([healsItself.done, stuck.done]).toEqual([true, true]);
|
||||
expect(healsItself.record.stagingMix!.retryBanner).toMatchObject({ firstShownAt: 0, shownMs: 60_000 });
|
||||
expect(stuck.record.stagingMix!.retryAfterSettle).toMatchObject({ accepted: true, issueStatusAtClick: "blocked" });
|
||||
expect(notes).toEqual(["staging_mix_settling", "staging_mix_retry_after_settle"]);
|
||||
expect(notes).toEqual(["staging_mix_settling", "staging_mix_retry_after_settle", "staging_mix_refreshed_after_settle"]);
|
||||
});
|
||||
|
||||
it("reports the three numbers first, counting the swap grader's own clicks and banners that healed by themselves", () => {
|
||||
it("follows a click on the retry budget even with a zero settle", async () => {
|
||||
const h = harness();
|
||||
h.issues.i2 = { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r2" } };
|
||||
const slow = h.observer("t2", "i2", {}, 12 * 60_000);
|
||||
await retryAfterSettleStage({ observers: [slow], api: { get: async <T>(path: string) => h.issues[path.split("/").pop()!] as T }, settleUntil: 0,
|
||||
cancelled: () => false, note: () => undefined, now: h.now, sleep: async ms => { h.advance(ms); } });
|
||||
expect(slow.clicks).toBe(1);
|
||||
expect(slow.done).toBe(true);
|
||||
expect(h.now()).toBeGreaterThanOrEqual(12 * 60_000);
|
||||
});
|
||||
|
||||
it("feeds the swap grader's reads into banner tracking", () => {
|
||||
const record = { issueId: "i1" } as TaskRecord;
|
||||
const other = { issueId: "i9" } as TaskRecord;
|
||||
const sink = bannerSink([{ plan: { stagingMix: task() } as TaskPlan, record }, { plan: {} as TaskPlan, record: other }]);
|
||||
sink("i1", { status: "blocked", interruption: { canRetry: true } }, 1_000);
|
||||
sink("i1", { status: "unreadable" }, 2_000);
|
||||
sink("i1", { status: "in_progress", interruption: null }, 31_000);
|
||||
sink("i9", { status: "blocked", interruption: { canRetry: true } }, 1_000);
|
||||
expect(record.stagingMix!.retryBanner).toMatchObject({ firstShownAt: 1_000, shownMs: 30_000, visibleSince: null });
|
||||
expect(other.stagingMix).toBeUndefined();
|
||||
});
|
||||
|
||||
it("counts done tasks with a verified deliverable and no click, and reports duplicates on their own line", () => {
|
||||
const record = (overrides: Partial<TaskRecord> = {}) => ({ issueId: null, retry: null, ...overrides }) as TaskRecord;
|
||||
const banner = { firstShownAt: 0, lastShownAt: 30_000, shownMs: 30_000, visibleSince: null, errorCode: "controller_lease_lost" };
|
||||
const ok = (name: string) => ({ name, ok: true }), bad = (name: string) => ({ name, ok: false });
|
||||
const clean = [ok("issue_done"), ok("ledger_rounds_verified"), ok("done_comment_posted"), ok("rounds_once"), ok("completion_once")];
|
||||
const result = summarizeStagingMix([
|
||||
{ taskId: "t1", issueId: "i1", record: record(), pass: true, finalStatus: "done" },
|
||||
{ taskId: "t2", issueId: "i2", record: record({ stagingMix: { retryBanner: banner } }), pass: true, finalStatus: "done" },
|
||||
{ taskId: "t3", issueId: "i3", record: record({ retry: { at: 1, accepted: true } as TaskRecord["retry"], stagingMix: { retryBanner: banner } }), pass: false, finalStatus: "done" },
|
||||
{ taskId: "t4", issueId: "i4", record: record({ stagingMix: { retryBanner: banner } }), pass: false, finalStatus: "blocked" },
|
||||
{ taskId: "t5", issueId: "i5", record: record(), pass: false, finalStatus: "in_progress" },
|
||||
], new Map([["i4", true]]), { violationCount: 2, violations: { nothing_pending: 2 }, agentsWithStalledSlots: 1, openTasks: 3 }, 60_000);
|
||||
{ taskId: "t1", issueId: "i1", record: record({ finalStatus: "blocked", timedOut: true }), finalStatus: "done", checks: clean },
|
||||
{ taskId: "t2", issueId: "i2", record: record({ stagingMix: { retryBanner: banner } }), finalStatus: "done", checks: [...clean.slice(0, 4), bad("completion_once")] },
|
||||
{ taskId: "t3", issueId: "i3", record: record({ retry: { at: 1, accepted: true } as TaskRecord["retry"], stagingMix: { retryBanner: banner } }), finalStatus: "done", checks: clean },
|
||||
{ taskId: "t4", issueId: "i4", record: record({ stagingMix: { retryBanner: banner } }), finalStatus: "blocked", checks: [bad("issue_done")] },
|
||||
{ taskId: "t5", issueId: "i5", record: record(), finalStatus: "done", checks: [ok("issue_done"), bad("ledger_rounds_verified")] },
|
||||
{ taskId: "t6", issueId: "i6", record: record(), finalStatus: "done", checks: null },
|
||||
], new Set(["i4"]), { violationCount: 2, violations: { nothing_pending: 2 }, agentsWithStalledSlots: 1, openTasks: 3 }, 60_000);
|
||||
expect(stagingMixHeadline(result)).toEqual([
|
||||
"completed hands-off / started: 2 / 5",
|
||||
"completed hands-off / started: 2 / 6",
|
||||
"duplicated side effects: 1 tasks",
|
||||
"Retry banner shown: 3 tasks",
|
||||
"Retry worked / Retry clicked: 1 / 2",
|
||||
]);
|
||||
expect(result.duplicatedSideEffects).toEqual([{ taskId: "t2", checks: ["completion_once"] }]);
|
||||
expect(result.bannerShown.find(entry => entry.taskId === "t2")).toMatchObject({ completedAnyway: true, errorCode: "controller_lease_lost", shownMs: 30_000 });
|
||||
expect(result.invariant.violationCount).toBe(2);
|
||||
});
|
||||
@@ -103,7 +146,7 @@ describe("staging-mix completion comments", () => {
|
||||
const nonce = "b3d70e77";
|
||||
const base = (comments: Array<{ body: string; runId: string | null }>): OracleArtifacts => ({ issueStatus: "done", documents: { ledger: ledger(nonce, 4) },
|
||||
agentComments: comments.map(comment => comment.body), files: {}, commands: {}, workspacePath: null,
|
||||
stagingMix: { followUp: null, followUpErrors: [], retries: [], comments } });
|
||||
stagingMix: collectStagingMixArtifacts({} as TaskRecord, comments.map(comment => ({ body: comment.body, authorAgentId: "a", createdByRunId: comment.runId }))) });
|
||||
const check = (artifacts: OracleArtifacts, name: string) => gradeStagingMix(nonce, task(), artifacts).checks.find(entry => entry.name === name)!;
|
||||
|
||||
it("counts only the DONE comment itself, not a closing summary that quotes it", () => {
|
||||
@@ -116,6 +159,13 @@ describe("staging-mix completion comments", () => {
|
||||
expect(check(base(comments), "completion_once").ok).toBe(true);
|
||||
expect(gradeStagingMix(nonce, task(), base(comments)).verdict).toBe("pass");
|
||||
expect(check(base([...comments, { body: ` \`${doneMarker(nonce)}\`. `, runId: "run-c" }]), "done_comment_once")).toMatchObject({ ok: false, detail: "comments=2" });
|
||||
expect(check(base(comments.slice(0, 2)), "done_comment_posted")).toMatchObject({ ok: false, detail: "comments=0" });
|
||||
});
|
||||
|
||||
it("marks completion_once unreadable instead of passing when the comments read failed", () => {
|
||||
const unread: OracleArtifacts = { ...base([{ body: doneMarker(nonce), runId: "run-b" }]), stagingMix: collectStagingMixArtifacts({} as TaskRecord, null) };
|
||||
expect(check(unread, "completion_once")).toMatchObject({ ok: false, detail: "unreadable: the task's comments could not be read" });
|
||||
expect(check(unread, "done_comment_posted").ok).toBe(true);
|
||||
});
|
||||
|
||||
it("flags a completion summary repeated by a different run as a duplicate completion", () => {
|
||||
|
||||
@@ -1,16 +1,25 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import type { ChaosApi } from "./api.js";
|
||||
import { bannerShownMs, noteRetryBanner } from "./staging-mix.js";
|
||||
import type { InterruptionSample, TaskPlan, TaskRecord, TaskSample } from "./types.js";
|
||||
import { bannerShownMs, DUPLICATE_CHECKS, noteRetryBanner } from "./staging-mix.js";
|
||||
import type { InterruptionSample, OracleResult, TaskPlan, TaskRecord, TaskSample } from "./types.js";
|
||||
|
||||
type Json = Record<string, any>;
|
||||
|
||||
export interface IssueView { status: string; canRetry: boolean; retryWait: boolean; observedRunId: string | null; interruption: InterruptionSample | null }
|
||||
export const RETRY_FOLLOW_MS = 15 * 60_000;
|
||||
|
||||
export function shouldClickRetryAfterSettle(view: Pick<IssueView, "status" | "canRetry" | "retryWait">, priorClicks: number): boolean {
|
||||
return view.status !== "done" && view.canRetry && !view.retryWait && priorClicks === 0;
|
||||
export interface IssueView {
|
||||
status: string;
|
||||
canRetry: boolean;
|
||||
retryWait: boolean;
|
||||
retryScheduled: boolean;
|
||||
observedRunId: string | null;
|
||||
interruption: InterruptionSample | null;
|
||||
}
|
||||
|
||||
export function shouldClickRetryAfterSettle(view: Pick<IssueView, "status" | "canRetry" | "retryWait" | "retryScheduled">, priorClicks: number): boolean {
|
||||
return view.status !== "done" && view.canRetry && !view.retryWait && !view.retryScheduled && priorClicks === 0;
|
||||
}
|
||||
|
||||
export function issueView(issue: Json | null): IssueView {
|
||||
@@ -19,6 +28,7 @@ export function issueView(issue: Json | null): IssueView {
|
||||
status: String(issue?.status ?? "unreadable"),
|
||||
canRetry: interruption?.canRetry === true,
|
||||
retryWait: Boolean(interruption?.retryWait),
|
||||
retryScheduled: Boolean(issue?.scheduledRetry) || interruption?.state === "retry_scheduled" || Boolean(interruption?.nextRetryAt),
|
||||
observedRunId: typeof interruption?.observedRunId === "string" ? interruption.observedRunId : null,
|
||||
interruption: interruption ? { state: String(interruption.state), canRetry: interruption.canRetry === true, retryWait: Boolean(interruption.retryWait),
|
||||
workspaceFailure: null, observedRunId: typeof interruption.observedRunId === "string" ? interruption.observedRunId : null,
|
||||
@@ -26,11 +36,26 @@ export function issueView(issue: Json | null): IssueView {
|
||||
};
|
||||
}
|
||||
|
||||
export function noteBannerFromIssue(record: TaskRecord, issue: Json | null, at: number): void {
|
||||
if (!issue || issue.status === "unreadable") return;
|
||||
const view = issueView(issue);
|
||||
noteRetryBanner(record.stagingMix ??= {}, view.status !== "done" && view.canRetry, at, null);
|
||||
}
|
||||
|
||||
export function bannerSink(observers: ReadonlyArray<{ plan: TaskPlan; record: TaskRecord }>): (issueId: string, issue: Json | null, at: number) => void {
|
||||
const byIssue = new Map(observers.flatMap(observer => observer.plan.stagingMix && observer.record.issueId ? [[observer.record.issueId, observer] as const] : []));
|
||||
return (issueId, issue, at) => {
|
||||
const observer = byIssue.get(issueId);
|
||||
if (observer) noteBannerFromIssue(observer.record, issue, at);
|
||||
};
|
||||
}
|
||||
|
||||
export interface RetryStageObserver {
|
||||
plan: TaskPlan;
|
||||
record: TaskRecord;
|
||||
done: boolean;
|
||||
poll(now?: number): Promise<void>;
|
||||
reopen(now: number): void;
|
||||
retryAfterSettle(sample: TaskSample, now: number): Promise<void>;
|
||||
}
|
||||
|
||||
@@ -38,15 +63,15 @@ export interface RetryStageInput {
|
||||
observers: readonly RetryStageObserver[];
|
||||
api: Pick<ChaosApi, "get">;
|
||||
settleUntil: number;
|
||||
settleMs: number;
|
||||
cancelled: () => boolean;
|
||||
note: (kind: string, data: Record<string, unknown>) => void;
|
||||
followMs?: number;
|
||||
pollMs?: number;
|
||||
now?: () => number;
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
}
|
||||
|
||||
export async function retryAfterSettleStage(input: RetryStageInput): Promise<{ clicked: string[] }> {
|
||||
export async function retryAfterSettleStage(input: RetryStageInput): Promise<{ clicked: string[]; refreshed: string[] }> {
|
||||
const now = input.now ?? Date.now;
|
||||
const sleep = input.sleep ?? (ms => delay(ms));
|
||||
const pollMs = input.pollMs ?? 60_000;
|
||||
@@ -55,9 +80,8 @@ export async function retryAfterSettleStage(input: RetryStageInput): Promise<{ c
|
||||
const read = async (observers: readonly RetryStageObserver[]) => {
|
||||
for (const observer of observers) {
|
||||
const issue = await input.api.get<Json>(`/api/issues/${observer.record.issueId}`, 10_000).catch(() => null);
|
||||
const view = issueView(issue);
|
||||
views.set(observer, view);
|
||||
noteRetryBanner(observer.record.stagingMix ??= {}, view.status !== "done" && view.canRetry, now(), null);
|
||||
views.set(observer, issueView(issue));
|
||||
noteBannerFromIssue(observer.record, issue, now());
|
||||
}
|
||||
};
|
||||
input.note("staging_mix_settling", { settleUntil: new Date(input.settleUntil).toISOString(), tasks: tracked.length });
|
||||
@@ -66,46 +90,66 @@ export async function retryAfterSettleStage(input: RetryStageInput): Promise<{ c
|
||||
if (input.cancelled() || now() >= input.settleUntil || tracked.every(observer => views.get(observer)?.status === "done")) break;
|
||||
await sleep(Math.min(pollMs, Math.max(1, input.settleUntil - now())));
|
||||
}
|
||||
const clicked: RetryStageObserver[] = [];
|
||||
const followed: RetryStageObserver[] = [];
|
||||
const clicked: string[] = [], refreshed: string[] = [];
|
||||
for (const observer of tracked) {
|
||||
const view = views.get(observer);
|
||||
if (!view || input.cancelled() || !shouldClickRetryAfterSettle(view, observer.record.retry ? 1 : 0)) continue;
|
||||
if (!view || input.cancelled()) continue;
|
||||
if (view.status === "done" && observer.record.finalStatus !== "done") {
|
||||
observer.reopen(now());
|
||||
refreshed.push(observer.plan.taskId);
|
||||
followed.push(observer);
|
||||
continue;
|
||||
}
|
||||
if (!shouldClickRetryAfterSettle(view, observer.record.retry ? 1 : 0)) continue;
|
||||
const at = now();
|
||||
await observer.retryAfterSettle({ at, reachable: true, serverReady: true, issueStatus: view.status, interruption: view.interruption }, at);
|
||||
(observer.record.stagingMix ??= {}).retryAfterSettle = { at, accepted: observer.record.retry?.accepted ?? null, issueStatusAtClick: view.status };
|
||||
input.note("staging_mix_retry_after_settle", { taskId: observer.plan.taskId, accepted: observer.record.retry?.accepted ?? null });
|
||||
clicked.push(observer);
|
||||
clicked.push(observer.plan.taskId);
|
||||
followed.push(observer);
|
||||
}
|
||||
const until = now() + input.settleMs;
|
||||
while (clicked.some(observer => !observer.done) && !input.cancelled() && now() < until) {
|
||||
for (const observer of clicked) if (!observer.done) await observer.poll(now());
|
||||
if (refreshed.length) input.note("staging_mix_refreshed_after_settle", { taskIds: refreshed });
|
||||
const until = now() + (input.followMs ?? RETRY_FOLLOW_MS);
|
||||
while (followed.some(observer => !observer.done) && !input.cancelled() && now() < until) {
|
||||
for (const observer of followed) if (!observer.done) await observer.poll(now());
|
||||
await sleep(Math.min(pollMs, 15_000));
|
||||
}
|
||||
return { clicked: clicked.map(observer => observer.plan.taskId) };
|
||||
return { clicked, refreshed };
|
||||
}
|
||||
|
||||
export interface ResultTask { taskId: string; issueId: string | null; record: TaskRecord; pass: boolean; finalStatus: string | null }
|
||||
export interface ResultTask { taskId: string; issueId: string | null; record: TaskRecord; finalStatus: string | null; checks: OracleResult["checks"] | null }
|
||||
|
||||
export interface StagingMixResult {
|
||||
started: number;
|
||||
completedHandsOff: number;
|
||||
duplicatedSideEffects: Array<{ taskId: string; checks: string[] }>;
|
||||
bannerShown: Array<{ taskId: string; firstShownAt: string; shownMs: number; errorCode: string | null; completedAnyway: boolean }>;
|
||||
retryClicked: number;
|
||||
retryWorked: number;
|
||||
invariant: { violationCount: number | null; violations: Record<string, number> | null; agentsWithStalledSlots: number | null; openTasks: number | null; error?: string };
|
||||
}
|
||||
|
||||
export function summarizeStagingMix(tasks: readonly ResultTask[], extraClicks: ReadonlyMap<string, boolean | null>, invariant: StagingMixResult["invariant"], now = Date.now()): StagingMixResult {
|
||||
export function deliverableVerified(checks: OracleResult["checks"] | null): boolean {
|
||||
return Boolean(checks?.length) && checks!.every(check => check.ok || DUPLICATE_CHECKS.has(check.name));
|
||||
}
|
||||
|
||||
export function summarizeStagingMix(tasks: readonly ResultTask[], extraClicks: ReadonlySet<string>, invariant: StagingMixResult["invariant"], now = Date.now()): StagingMixResult {
|
||||
const clicked = (task: ResultTask) => Boolean(task.record.retry || task.record.priorRetries?.length || (task.issueId && extraClicks.has(task.issueId)));
|
||||
const completed = (task: ResultTask) => task.finalStatus === "done" && deliverableVerified(task.checks);
|
||||
const clickedTasks = tasks.filter(clicked);
|
||||
return {
|
||||
started: tasks.length,
|
||||
completedHandsOff: tasks.filter(task => task.pass && !clicked(task)).length,
|
||||
completedHandsOff: tasks.filter(task => completed(task) && !clicked(task)).length,
|
||||
duplicatedSideEffects: tasks.flatMap(task => {
|
||||
const failing = (task.checks ?? []).filter(check => !check.ok && DUPLICATE_CHECKS.has(check.name)).map(check => check.name);
|
||||
return failing.length ? [{ taskId: task.taskId, checks: failing }] : [];
|
||||
}),
|
||||
bannerShown: tasks.flatMap(task => task.record.stagingMix?.retryBanner ? [{ taskId: task.taskId, firstShownAt: new Date(task.record.stagingMix.retryBanner.firstShownAt).toISOString(),
|
||||
shownMs: bannerShownMs(task.record.stagingMix.retryBanner, now), errorCode: task.record.stagingMix.retryBanner.errorCode,
|
||||
completedAnyway: task.finalStatus === "done" && !clicked(task) }] : []),
|
||||
retryClicked: clickedTasks.length,
|
||||
retryWorked: clickedTasks.filter(task => task.finalStatus === "done").length,
|
||||
retryWorked: clickedTasks.filter(completed).length,
|
||||
invariant,
|
||||
};
|
||||
}
|
||||
@@ -113,6 +157,7 @@ export function summarizeStagingMix(tasks: readonly ResultTask[], extraClicks: R
|
||||
export function stagingMixHeadline(result: StagingMixResult): string[] {
|
||||
return [
|
||||
`completed hands-off / started: ${result.completedHandsOff} / ${result.started}`,
|
||||
`duplicated side effects: ${result.duplicatedSideEffects.length} tasks`,
|
||||
`Retry banner shown: ${result.bannerShown.length} tasks`,
|
||||
`Retry worked / Retry clicked: ${result.retryWorked} / ${result.retryClicked}`,
|
||||
];
|
||||
@@ -130,15 +175,15 @@ export async function readAssignedWorkInvariant(api: Pick<ChaosApi, "get">, comp
|
||||
}
|
||||
|
||||
export async function writeStagingMixResults(input: { waveDir: string; api: Pick<ChaosApi, "get">; companyId: string;
|
||||
tasks: ReadonlyArray<{ plan: TaskPlan; record: TaskRecord }>; passed: ReadonlyMap<string, boolean>;
|
||||
timeline: ReadonlyArray<{ kind: string; [key: string]: unknown }> }): Promise<StagingMixResult> {
|
||||
const extraClicks = new Map<string, boolean | null>(input.timeline.filter(entry => entry.kind === "release_swap_retry_clicked" && typeof entry.issueId === "string")
|
||||
.map(entry => [String(entry.issueId), entry.accepted === true ? true : entry.accepted === false ? false : null]));
|
||||
tasks: ReadonlyArray<{ plan: TaskPlan; record: TaskRecord }>; timeline: ReadonlyArray<{ kind: string; [key: string]: unknown }> }): Promise<StagingMixResult> {
|
||||
const extraClicks = new Set(input.timeline.filter(entry => entry.kind === "release_swap_retry_clicked" && typeof entry.issueId === "string").map(entry => String(entry.issueId)));
|
||||
const tasks: ResultTask[] = [];
|
||||
for (const task of input.tasks) {
|
||||
const issue = task.record.issueId ? await input.api.get<Json>(`/api/issues/${task.record.issueId}`, 10_000).catch(() => null) : null;
|
||||
tasks.push({ taskId: task.plan.taskId, issueId: task.record.issueId, record: task.record, pass: input.passed.get(task.plan.taskId) === true,
|
||||
finalStatus: issue?.status ?? task.record.finalStatus });
|
||||
const grade = await readFile(path.join(input.waveDir, `${task.plan.taskId}.grade.json`), "utf8")
|
||||
.then(text => JSON.parse(text) as { grade?: { T?: OracleResult } }).catch(() => null);
|
||||
tasks.push({ taskId: task.plan.taskId, issueId: task.record.issueId, record: task.record, finalStatus: issue?.status ?? task.record.finalStatus,
|
||||
checks: grade?.grade?.T?.checks ?? null });
|
||||
}
|
||||
const result = summarizeStagingMix(tasks, extraClicks, await readAssignedWorkInvariant(input.api, input.companyId));
|
||||
await writeFile(path.join(input.waveDir, "staging-mix-results.json"), `${JSON.stringify({ schema: "paperclip.restart-chaos.staging-mix/v1",
|
||||
@@ -146,6 +191,7 @@ export async function writeStagingMixResults(input: { waveDir: string; api: Pick
|
||||
const invariant = result.invariant.violationCount === null ? `unreadable${result.invariant.error ? ` (${result.invariant.error})` : ""}`
|
||||
: `${result.invariant.violationCount} violations, ${result.invariant.agentsWithStalledSlots} agents with stalled slots, ${result.invariant.openTasks} open tasks`;
|
||||
await writeFile(path.join(input.waveDir, "staging-mix-summary.md"), [...stagingMixHeadline(result), `Assigned-work invariant: ${invariant}`, "",
|
||||
...result.duplicatedSideEffects.map(entry => `- ${entry.taskId}: duplicated ${entry.checks.join(", ")}`),
|
||||
...result.bannerShown.map(entry => `- ${entry.taskId}: Retry banner from ${entry.firstShownAt} for ${Math.round(entry.shownMs / 1000)} s`
|
||||
+ `${entry.errorCode ? ` (${entry.errorCode})` : ""}${entry.completedAnyway ? ", completed without a click" : ""}`), ""].join("\n"), { mode: 0o600 });
|
||||
return result;
|
||||
|
||||
@@ -145,7 +145,7 @@ export class StagingMixRun {
|
||||
|
||||
async retryAfterSettle(observers: readonly RetryStageObserver[], api: Pick<ChaosApi, "get">, cancelled: () => boolean): Promise<void> {
|
||||
if (this.mix.retryDuringWave) return;
|
||||
await retryAfterSettleStage({ observers, api, settleUntil: this.settleUntil(), settleMs: this.mix.settleMinutes * 60_000, cancelled, note: this.note });
|
||||
await retryAfterSettleStage({ observers, api, settleUntil: this.settleUntil(), cancelled, note: this.note });
|
||||
}
|
||||
|
||||
mayLaunch(now: number, next: TaskPlan): boolean {
|
||||
|
||||
@@ -63,8 +63,11 @@ export interface StagingMixArtifacts {
|
||||
followUpErrors: string[];
|
||||
retries: NonNullable<StagingMixRecord["retries"]>;
|
||||
comments?: Array<{ body: string; runId: string | null }>;
|
||||
commentsUnreadable?: boolean;
|
||||
}
|
||||
|
||||
export const DUPLICATE_CHECKS: ReadonlySet<string> = new Set(["rounds_once", "done_comment_once", "completion_once", "subtasks_once", "subtask_rounds_once"]);
|
||||
|
||||
export const STAGING_MIX_ARG_OPTIONS = {
|
||||
workload: { type: "string" },
|
||||
"mix-runs": { type: "string" },
|
||||
@@ -412,10 +415,12 @@ export function gradeStagingMix(nonce: string, task: StagingMixTask, artifacts:
|
||||
const checks: OracleResult["checks"] = [
|
||||
{ name: "issue_done", ok: artifacts.issueStatus === "done", detail: `status=${artifacts.issueStatus ?? "unknown"}` },
|
||||
{ name: "ledger_rounds_verified", ok: own.verified, detail: own.detail },
|
||||
{ name: "done_comment_posted", ok: doneComments >= 1, detail: `comments=${doneComments}` },
|
||||
{ name: "rounds_once", ok: own.duplicates === 0, detail: `duplicates=${own.duplicates}` },
|
||||
{ name: "done_comment_once", ok: doneComments === 1, detail: `comments=${doneComments}` },
|
||||
{ name: "completion_once", ok: completionRuns.length <= 1,
|
||||
detail: completionRuns.length > 1 ? `duplicate completion: runs ${completionRuns.map(id => id.slice(0, 8)).join(", ")} each reported completion` : `runs=${completionRuns.length}` },
|
||||
{ name: "done_comment_once", ok: doneComments <= 1, detail: `comments=${doneComments}` },
|
||||
artifacts.stagingMix?.commentsUnreadable ? { name: "completion_once", ok: false, detail: "unreadable: the task's comments could not be read" }
|
||||
: { name: "completion_once", ok: completionRuns.length <= 1,
|
||||
detail: completionRuns.length > 1 ? `duplicate completion: runs ${completionRuns.map(id => id.slice(0, 8)).join(", ")} each reported completion` : `runs=${completionRuns.length}` },
|
||||
];
|
||||
if (task.followUpAfterMs !== null) {
|
||||
const errors = artifacts.stagingMix?.followUpErrors ?? [];
|
||||
@@ -427,16 +432,20 @@ export function gradeStagingMix(nonce: string, task: StagingMixTask, artifacts:
|
||||
detail: `posted ${new Date(followUp.at).toISOString()}${followUp.inOverlap ? " in overlap" : ""}` });
|
||||
}
|
||||
if (task.subtasks > 0) {
|
||||
checks.push({ name: "subtasks_once", ok: children.length === task.subtasks, detail: `subtasks=${children.length}/${task.subtasks}` });
|
||||
checks.push({ name: "subtasks_once", ok: children.length <= task.subtasks, detail: `subtasks=${children.length}/${task.subtasks}` });
|
||||
let childDuplicates = 0;
|
||||
for (let part = 1; part <= task.subtasks; part += 1) {
|
||||
const nonceOf = childNonce(nonce, part);
|
||||
const child = children.filter(candidate => (candidate.title ?? "").includes(nonceOf));
|
||||
const result = verifyLedger(nonceOf, SUBTASK_ROUNDS, child[0]?.documents.ledger);
|
||||
const results = child.map(candidate => ({ candidate, result: verifyLedger(nonceOf, SUBTASK_ROUNDS, candidate.documents.ledger) }));
|
||||
const best = results.find(entry => entry.candidate.status === "done" && entry.result.verified) ?? results[0];
|
||||
childDuplicates += results.reduce((sum, entry) => sum + entry.result.duplicates, 0);
|
||||
const lastHash = expectedRoundHash(nonceOf, SUBTASK_ROUNDS);
|
||||
checks.push({ name: `subtask_${part}_done`, ok: child.length === 1 && child[0]!.status === "done" && result.verified && result.duplicates === 0,
|
||||
detail: child.length ? `status=${child[0]!.status} ${result.detail}` : "missing" });
|
||||
checks.push({ name: `subtask_${part}_done`, ok: Boolean(best && best.candidate.status === "done" && best.result.verified),
|
||||
detail: best ? `status=${best.candidate.status} ${best.result.detail}` : "missing" });
|
||||
checks.push({ name: `parent_read_subtask_${part}`, ok: (ledger ?? "").includes(childLine(part, lastHash)), detail: childLine(part, lastHash) });
|
||||
}
|
||||
checks.push({ name: "subtask_rounds_once", ok: childDuplicates === 0, detail: `duplicates=${childDuplicates}` });
|
||||
}
|
||||
return { verdict: checks.every(check => check.ok) ? "pass" : "fail", checks };
|
||||
}
|
||||
@@ -561,8 +570,8 @@ export function bannerShownMs(banner: RetryBanner, now: number): number {
|
||||
return banner.shownMs + (banner.visibleSince === null ? 0 : now - banner.visibleSince);
|
||||
}
|
||||
|
||||
export async function collectStagingMixArtifacts(api: Pick<ChaosApi, "get">, issueId: string, record: TaskRecord): Promise<StagingMixArtifacts> {
|
||||
const rows = await api.get<Array<{ body?: string; authorAgentId?: string | null; createdByRunId?: string | null }>>(`/api/issues/${issueId}/comments`).catch(() => null);
|
||||
export function collectStagingMixArtifacts(record: TaskRecord, rows: ReadonlyArray<{ body?: unknown; authorAgentId?: unknown; createdByRunId?: unknown }> | null): StagingMixArtifacts {
|
||||
return { followUp: record.stagingMix?.followUp ?? null, followUpErrors: record.stagingMix?.followUpErrors ?? [], retries: record.stagingMix?.retries ?? [],
|
||||
...(rows ? { comments: rows.filter(row => row.authorAgentId).map(row => ({ body: String(row.body ?? ""), runId: row.createdByRunId ?? null })) } : {}) };
|
||||
...(rows ? { comments: rows.filter(row => row.authorAgentId).map(row => ({ body: String(row.body ?? ""), runId: typeof row.createdByRunId === "string" ? row.createdByRunId : null })) }
|
||||
: { commentsUnreadable: true }) };
|
||||
}
|
||||
Reference in new issue
Block a user