mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-17 04:21:24 +01:00
test(restart-chaos): grade staging-mix hands-off and click Retry once after the settle
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
cdefac664b
commit
ddb64d3223
9 files changed
+419
-27
No files matched your search
@@ -1338,6 +1338,10 @@ node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona
|
||||
node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona --daytona-image $IMG --daytona-warm --daytona-cpu 1 \
|
||||
--storage s3 --ephemeral-server --workload staging-mix --faults K15 --k15-overlap-seconds 480-540 --mix-runs 15 --mix-queue 3 \
|
||||
--mix-minutes 40 --mix-fault-spacing-minutes 12 --mix-turn-minutes 5-15 --budget-cents <cap> --seed 7102
|
||||
# The same short form locally (agents on this Mac, so 10 runs). `--ephemeral-server` runs only on Daytona, so it is left out.
|
||||
node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env local --storage s3 --workload staging-mix --faults K15 \
|
||||
--k15-overlap-seconds 480-540 --mix-runs 10 --mix-queue 3 --mix-minutes 40 --mix-fault-spacing-minutes 12 --mix-turn-minutes 5-15 \
|
||||
--budget-cents 5000 --seed 7201
|
||||
# Full form (about 4.5 hours): 6 swaps 40 minutes apart, 15-45 minute turns, 30 runs and a queue of 6.
|
||||
RESTART_CHAOS_DAYTONA_VCPU_BUDGET=<org headroom> node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona \
|
||||
--daytona-image $IMG --daytona-warm --daytona-cpu 1 --storage s3 --ephemeral-server --workload staging-mix --faults K15 \
|
||||
@@ -1381,13 +1385,29 @@ one settled and at least one task is in flight; it hits every task in flight (`s
|
||||
tasks do: the wave's clock crosses them. A slot that never fired because every task rested first fails the wave as a
|
||||
harness error (`staging_mix_fault_slots_missed`). With `--faults K0` the tasks are `K0` and nothing fires.
|
||||
|
||||
**Follow-ups and Retries.** About half of the tasks (`--mix-follow-up-share`) get a board comment while a run is
|
||||
running. The comment asks the agent to add `FOLLOWUP-<nonce>` to the ledger. A failed task's Retry is clicked the
|
||||
same way. Both are held until the next overlap window, but for at most `--mix-hold-minutes` (default 10); a follow-up
|
||||
also goes out before the last 15% of the planned turn. A failed post is tried again on the next poll. The
|
||||
overlap window comes from `setOverlapProbe` in `staging-mix.ts`, which a swap driver registers. Without a probe, the
|
||||
window is any time a fault is in progress and the server still answers (a K15 overlap). The timeline records
|
||||
`staging_mix_follow_up` and `staging_mix_retry_in_overlap`.
|
||||
**Follow-ups.** About half of the tasks (`--mix-follow-up-share`) get a board comment while a run is running. The
|
||||
comment asks the agent to add `FOLLOWUP-<nonce>` to the ledger. It is held until the next overlap window, but for at
|
||||
most `--mix-hold-minutes` (default 10), and it goes out before the last 15% of the planned turn. A failed post is
|
||||
tried again on the next poll with the same `clientRequestId`. The overlap window comes from `setOverlapProbe` in
|
||||
`staging-mix.ts`, which a swap driver registers. Without a probe, the window is any time a fault is in progress and the
|
||||
server still answers (a K15 overlap). The timeline records `staging_mix_follow_up`.
|
||||
|
||||
**Retries: hands-off first.** By default the harness never clicks Retry during the wave or the settle, so the product's
|
||||
own continuation is what gets measured. A task that shows a Retry banner is kept under watch, not rested. Every banner
|
||||
is recorded per task: when it first showed, how long it stayed, and the error code behind it, even when the task then
|
||||
finished by itself. A banner is a defect, because the product goal is that it never appears. After the settle
|
||||
(`--mix-settle-minutes`, default 15, after the last fault slot ends), each task that is still not done and still
|
||||
offers Retry gets one click (`staging_mix_retry_after_settle`), and the harness follows it to rest. When the release-swap
|
||||
cell drives the wave, its own end-of-wave pass makes that one click instead. Both paths use
|
||||
`shouldClickRetryAfterSettle` in `staging-mix-retry.ts`: never on a done task, a task waiting on a scheduled retry, or a
|
||||
task that was already clicked. So no task gets two clicks. `--mix-retry-during-wave` restores the old behaviour: Retries
|
||||
clicked during the wave, held until an overlap window as follow-ups are (`staging_mix_retry_in_overlap`).
|
||||
|
||||
**Results.** `staging-mix-results.json` and the first lines of `SUMMARY.md` give three numbers, in this order:
|
||||
`completed hands-off / started`, `Retry banner shown: N tasks` and `Retry worked / Retry clicked`. A Retry counts as
|
||||
worked when the task is done at the end. The banner list follows, then one read of
|
||||
`GET /api/companies/:id/assigned-work-invariant` at the end of the wave: `violationCount`, `violations` by reason,
|
||||
`agentsWithStalledSlots` and `openTasks`.
|
||||
|
||||
**Grade.** The usual R1–R6, U1–U5 and T rules apply. A needed Retry still fails R3. The T-oracle checks the result,
|
||||
not the status:
|
||||
@@ -1397,7 +1417,8 @@ not the status:
|
||||
| `issue_done` | the task is not `done` |
|
||||
| `ledger_rounds_verified` | a round is missing, ran under 80 s, or its hash differs from the harness's own computation (POSIX `cksum`, chained per tick) |
|
||||
| `rounds_once` | a round line appears twice (a tool ran again) |
|
||||
| `done_comment_once` | the `DONE-<nonce>` comment is missing or was posted twice |
|
||||
| `done_comment_once` | the comment whose whole body is `DONE-<nonce>` is missing or was posted twice; a summary that quotes the marker does not count |
|
||||
| `completion_once` | more than one run posted a comment that mentions `DONE-<nonce>` (a duplicate completion) |
|
||||
| `follow_up_posted` | a planned follow-up was never posted (the detail lists the post errors) |
|
||||
| `follow_up_served` | a follow-up was posted during a run and the ledger lacks its line |
|
||||
| `subtasks_once`, `subtask_<k>_done`, `parent_read_subtask_<k>` | a parent created more or fewer than two subtasks, a subtask's ledger fails, or the parent did not report a subtask's last hash |
|
||||
|
||||
@@ -49,6 +49,7 @@ import { chaosLifecycleOverrides, seedChaosLab } from "./seed.js";
|
||||
import { ChaosSupervisor } from "./supervisor.js";
|
||||
import { planStagingMixWave } from "./staging-mix.js";
|
||||
import { StagingMixRun, stagingMixSeedOptions } from "./staging-mix-run.js";
|
||||
import { writeStagingMixResults } from "./staging-mix-retry.js";
|
||||
import { TimeShiftedApi } from "./time-faults.js";
|
||||
import type { RestartStep } from "./repeated-restarts.js";
|
||||
import { isBrokenStdoutError, markTimelineEchoBroken, Timeline } from "./timeline.js";
|
||||
@@ -734,13 +735,8 @@ async function main(): Promise<number> {
|
||||
sandboxWatchStopped = true;
|
||||
await mix?.stop();
|
||||
await Promise.all(faultWork);
|
||||
const mixShortfall = mix && !interrupted ? mix.shortfall(launched.length) : [];
|
||||
if (mixShortfall.length) {
|
||||
timeline.add("staging_mix_shortfall", { problems: mixShortfall, unfired: mix!.unfiredSlots() });
|
||||
harnessError ??= new Error(`Staging mix tested less than planned: ${mixShortfall.join("; ")}`);
|
||||
}
|
||||
if (mix) console.info(`Staging mix: ${launched.length} tasks started; ${mix.unlaunched} planned tasks never launched (excluded from tasks started); `
|
||||
+ `fault slots fired ${mix.slotCoverage.map(entry => `${Math.round(entry.atMs / 60_000)}m:${entry.kind}×${entry.exposed} tasks`).join(", ") || "none"} of ${mix.slots.length}`);
|
||||
harnessError ??= mix && !interrupted ? mix.report(launched.length) ?? undefined : undefined;
|
||||
if (mix && !interrupted && !releaseSwap) await mix.retryAfterSettle(launched, api, () => interrupted);
|
||||
if (storageTrigger) {
|
||||
const shortfall = storageTriggerShortfall(storageTrigger.finish(faults));
|
||||
storageSection = storageTriggerMarkdown(storageTrigger.receipts);
|
||||
@@ -840,6 +836,8 @@ async function main(): Promise<number> {
|
||||
rules: Object.fromEntries(Object.entries(grade.rules).map(([id, rule]) => [id, rule.verdict])), T: grade.T.verdict,
|
||||
universal: Object.fromEntries(Object.entries(grade.universal ?? {}).map(([id, rule]) => [id, `${rule.verdict}${rule.enforced ? "" : " (warning)"}`])) });
|
||||
}
|
||||
if (mix) await writeStagingMixResults({ waveDir, api, companyId: lab.companyId, tasks: launched, passed: new Map(lines.map(line => [line.taskId, line.pass])), timeline: timeline.entries })
|
||||
.catch(error => { harnessError ??= new Error(`Staging-mix results failed: ${String(error)}`); });
|
||||
// MS1–MS3: the cross-server invariant over the whole wave (multi-server.ts).
|
||||
if (faults.some(receipt => receipt.multiServer)) {
|
||||
const crossServer = await readCrossServerEvidence(evidenceDb, lab.companyId)
|
||||
@@ -987,7 +985,7 @@ async function main(): Promise<number> {
|
||||
if (lines.length) {
|
||||
await appendLedger(resultsRoot, lines);
|
||||
const summary = await writeSummary(resultsRoot, plan.waveId);
|
||||
await writeFile(path.join(waveDir, "SUMMARY.md"), [await readFile(path.join(waveDir, "release-swap-summary.md"), "utf8").catch(() => null), summary, warmShutdownSection, speedSection, storageSection].filter(Boolean).join("\n"), { mode: 0o600 });
|
||||
await writeFile(path.join(waveDir, "SUMMARY.md"), [...await Promise.all(["staging-mix-summary.md", "release-swap-summary.md"].map(file => readFile(path.join(waveDir, file), "utf8").catch(() => null))), summary, warmShutdownSection, speedSection, storageSection].filter(Boolean).join("\n"), { mode: 0o600 });
|
||||
console.info(`\n${summary}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ import { awaitingQuestionAnswer, collectQuestionArtifacts, questionStep } from "
|
||||
import { isSessionFault, rotateAgentSession } from "./session-faults.js";
|
||||
import { awaitingRewake, collectRewakeArtifacts, isRewakeWorkload, rewakeStep } from "./rewake.js";
|
||||
import { withLongTurn } from "./long-turn.js";
|
||||
import { holdsRetry, stagingMixArtifacts, stagingMixDefinition, stagingMixStep } from "./staging-mix.js";
|
||||
import { collectStagingMixArtifacts, holdsRetry, stagingMixDefinition, stagingMixStep } from "./staging-mix.js";
|
||||
import { captureRemoteWorkspaceFiles, readRemoteWorkspaceFile } from "./remote-workspace-files.js";
|
||||
import { taskFaultSchedule } from "./scheduler.js";
|
||||
import { g1ChildTitle, g1Decision, workloadDefinitions, type WorkloadAgents } from "./workloads.js";
|
||||
@@ -686,8 +686,9 @@ export class TaskObserver {
|
||||
}
|
||||
if (!await this.clearDiskFullAtRest(now)) { this.restCandidateSince = null; return; }
|
||||
if (status !== "done" && (!this.record.retry || faultSinceRetry) && sample.interruption?.canRetry) {
|
||||
if (this.plan.stagingMix && holdsRetry(this.record, this.plan.stagingMix.holdMs, now)) { this.restCandidateSince = null; return; }
|
||||
await this.boardRetry(sample, now);
|
||||
if (this.plan.stagingMix?.retryDuringWave && holdsRetry(this.record, this.plan.stagingMix.holdMs, now)) { this.restCandidateSince = null; return; }
|
||||
if (!this.plan.stagingMix || this.plan.stagingMix.retryDuringWave) { await this.boardRetry(sample, now); return; }
|
||||
this.restCandidateSince = null;
|
||||
return;
|
||||
}
|
||||
this.record.restAt = this.restCandidateSince ?? now;
|
||||
@@ -697,6 +698,14 @@ export class TaskObserver {
|
||||
canRetry: sample.interruption?.canRetry ?? false, retried: Boolean(this.record.retry) });
|
||||
}
|
||||
|
||||
async retryAfterSettle(sample: TaskSample, now: number): Promise<void> {
|
||||
this.done = false;
|
||||
this.record.restAt = null;
|
||||
this.record.timedOut = false;
|
||||
await this.boardRetry(sample, now);
|
||||
this.record.deadlineAt = now + RETRY_BUDGET_MS;
|
||||
}
|
||||
|
||||
private async boardRetry(sample: TaskSample, now: number): Promise<void> {
|
||||
if (this.record.retry) (this.record.priorRetries ??= []).push(this.record.retry);
|
||||
this.record.retry = await this.retry(sample, now);
|
||||
@@ -1092,7 +1101,7 @@ export class TaskObserver {
|
||||
status: String(run.status), errorCode: run.errorCode ?? null, createdAt: run.createdAt ?? null, finishedAt: run.finishedAt ?? null }))).catch(() => []),
|
||||
...(this.journey ? { journey: await this.journey.collect(issueId).finally(() => this.journey!.close()) } : {}),
|
||||
...(this.plan.workload === "H2" ? { handoff: await this.collectHandoff(issueId, issue) } : {}),
|
||||
...(this.plan.stagingMix ? { stagingMix: stagingMixArtifacts(this.record) } : {}),
|
||||
...(this.plan.stagingMix ? { stagingMix: await collectStagingMixArtifacts(this.api, issueId, this.record) } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
gradeReleaseSwapTask, overlappingRuns, releaseSwapMarkdown, summarizeGrades, swapMetrics,
|
||||
type DuplicateFacts, type GradeTaskInput, type RetryClick, type SwapRunFact, type TaskGrade,
|
||||
} from "./release-swap-grade.js";
|
||||
import { issueView, shouldClickRetryAfterSettle } from "./staging-mix-retry.js";
|
||||
import type { TimelineEntry } from "./timeline.js";
|
||||
import type { Db } from "../../packages/db/src/index.js";
|
||||
|
||||
@@ -208,7 +209,7 @@ export async function finishReleaseSwap(input: ReleaseSwapFinishInput): Promise<
|
||||
for (const issue of issues) {
|
||||
const id = String(issue.id);
|
||||
const current = views.get(id)!;
|
||||
if (current.status === "done" || current.interruption?.canRetry !== true || current.interruption?.retryWait) continue;
|
||||
if (!shouldClickRetryAfterSettle(issueView(current), clicks.get(id)?.length ?? 0)) continue;
|
||||
const click = await clickRetry(input, id, text(issue.identifier) ?? id, text(current.interruption?.observedRunId));
|
||||
clicks.set(id, [...(clicks.get(id) ?? []), click]);
|
||||
retried.push(id);
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseCommand } from "./config.js";
|
||||
import { doneMarker, expectedRoundHash, gradeStagingMix, planStagingMixWave, ROUND_SECONDS, type StagingMixTask } from "./staging-mix.js";
|
||||
import {
|
||||
readAssignedWorkInvariant, retryAfterSettleStage, shouldClickRetryAfterSettle, stagingMixHeadline, summarizeStagingMix, type RetryStageObserver,
|
||||
} from "./staging-mix-retry.js";
|
||||
import type { OracleArtifacts, TaskPlan, TaskRecord, TaskSample } from "./types.js";
|
||||
|
||||
const wave = (argv: string[]) => {
|
||||
const command = parseCommand(argv);
|
||||
if (command.mode !== "wave") throw new Error("expected a wave");
|
||||
return command.options;
|
||||
};
|
||||
|
||||
const task = (overrides: Partial<StagingMixTask> = {}): StagingMixTask => ({
|
||||
turnMinutes: 5, rounds: 4, subtasks: 0, followUpAfterMs: null, arriveAtMs: 0, holdMs: 600_000, ...overrides,
|
||||
});
|
||||
|
||||
const ledger = (nonce: string, rounds: number) => Array.from({ length: rounds }, (_, index) =>
|
||||
`ROUND ${index + 1} ${1_800_000_000 + index * 100} ${1_800_000_000 + index * 100 + ROUND_SECONDS} ${expectedRoundHash(nonce, index + 1)}`).join("\n");
|
||||
|
||||
describe("staging-mix Retry policy", () => {
|
||||
it("is hands-off by default, and keeps the overlap Retry only behind --mix-retry-during-wave", () => {
|
||||
const base = ["--workload", "staging-mix", "--faults", "K0", "--mix-turn-minutes", "5-15", "--mix-minutes", "40"];
|
||||
const handsOff = wave(base);
|
||||
expect(handsOff.stagingMix).toMatchObject({ retryDuringWave: false, settleMinutes: 15 });
|
||||
expect(planStagingMixWave("w", handsOff).tasks.every(entry => entry.stagingMix!.retryDuringWave === undefined)).toBe(true);
|
||||
const old = wave([...base, "--mix-retry-during-wave", "--mix-settle-minutes", "5"]);
|
||||
expect(old.stagingMix).toMatchObject({ retryDuringWave: true, settleMinutes: 5 });
|
||||
expect(planStagingMixWave("w", old).tasks.every(entry => entry.stagingMix!.retryDuringWave === true)).toBe(true);
|
||||
});
|
||||
|
||||
it("clicks once after the settle, only on a task still offering Retry that nobody clicked", () => {
|
||||
expect(shouldClickRetryAfterSettle({ status: "blocked", canRetry: true, retryWait: false }, 0)).toBe(true);
|
||||
expect(shouldClickRetryAfterSettle({ status: "blocked", canRetry: true, retryWait: false }, 1)).toBe(false);
|
||||
expect(shouldClickRetryAfterSettle({ status: "done", canRetry: true, retryWait: false }, 0)).toBe(false);
|
||||
expect(shouldClickRetryAfterSettle({ status: "blocked", canRetry: true, retryWait: true }, 0)).toBe(false);
|
||||
expect(shouldClickRetryAfterSettle({ status: "in_progress", canRetry: false, retryWait: false }, 0)).toBe(false);
|
||||
});
|
||||
|
||||
it("waits out the settle hands-off, then clicks once per stuck task and follows it to rest", async () => {
|
||||
let clock = 0;
|
||||
const issues: Record<string, Record<string, unknown>> = {
|
||||
i1: { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r1" } },
|
||||
i2: { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r2" } },
|
||||
i3: { status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r3" } },
|
||||
};
|
||||
const observer = (taskId: string, issueId: string, retry: TaskRecord["retry"] = null): RetryStageObserver & { clicks: number } => {
|
||||
const self = {
|
||||
plan: { taskId, stagingMix: task() } as TaskPlan, record: { issueId, retry } as TaskRecord, done: true, clicks: 0,
|
||||
async poll() { if (issues[issueId]!.status === "done") self.done = true; },
|
||||
async retryAfterSettle(sample: TaskSample, at: number) {
|
||||
self.done = false; self.clicks += 1;
|
||||
expect(sample.interruption?.observedRunId).toBe(`r${issueId.slice(1)}`);
|
||||
self.record.retry = { at, accepted: true } as TaskRecord["retry"];
|
||||
issues[issueId] = { status: "done" };
|
||||
},
|
||||
};
|
||||
return self;
|
||||
};
|
||||
const healsItself = observer("t1", "i1"), stuck = observer("t2", "i2"), alreadyClicked = observer("t3", "i3", { at: 1, accepted: true } as TaskRecord["retry"]);
|
||||
const notes: string[] = [];
|
||||
const result = await retryAfterSettleStage({ observers: [healsItself, stuck, alreadyClicked], api: { get: async <T>(path: string) => issues[path.split("/").pop()!] as T },
|
||||
settleUntil: 120_000, settleMs: 60_000, cancelled: () => false, note: kind => notes.push(kind), pollMs: 60_000, now: () => clock,
|
||||
sleep: async ms => { clock += ms; if (clock >= 60_000) issues.i1 = { status: "done" }; } });
|
||||
expect(clock).toBeGreaterThanOrEqual(120_000);
|
||||
expect(result.clicked).toEqual(["t2"]);
|
||||
expect([healsItself.clicks, stuck.clicks, alreadyClicked.clicks]).toEqual([0, 1, 0]);
|
||||
expect(stuck.done).toBe(true);
|
||||
expect(healsItself.record.stagingMix!.retryBanner).toMatchObject({ firstShownAt: 0, shownMs: 60_000 });
|
||||
expect(stuck.record.stagingMix!.retryAfterSettle).toMatchObject({ accepted: true, issueStatusAtClick: "blocked" });
|
||||
expect(notes).toEqual(["staging_mix_settling", "staging_mix_retry_after_settle"]);
|
||||
});
|
||||
|
||||
it("reports the three numbers first, counting the swap grader's own clicks and banners that healed by themselves", () => {
|
||||
const record = (overrides: Partial<TaskRecord> = {}) => ({ issueId: null, retry: null, ...overrides }) as TaskRecord;
|
||||
const banner = { firstShownAt: 0, lastShownAt: 30_000, shownMs: 30_000, visibleSince: null, errorCode: "controller_lease_lost" };
|
||||
const result = summarizeStagingMix([
|
||||
{ taskId: "t1", issueId: "i1", record: record(), pass: true, finalStatus: "done" },
|
||||
{ taskId: "t2", issueId: "i2", record: record({ stagingMix: { retryBanner: banner } }), pass: true, finalStatus: "done" },
|
||||
{ taskId: "t3", issueId: "i3", record: record({ retry: { at: 1, accepted: true } as TaskRecord["retry"], stagingMix: { retryBanner: banner } }), pass: false, finalStatus: "done" },
|
||||
{ taskId: "t4", issueId: "i4", record: record({ stagingMix: { retryBanner: banner } }), pass: false, finalStatus: "blocked" },
|
||||
{ taskId: "t5", issueId: "i5", record: record(), pass: false, finalStatus: "in_progress" },
|
||||
], new Map([["i4", true]]), { violationCount: 2, violations: { nothing_pending: 2 }, agentsWithStalledSlots: 1, openTasks: 3 }, 60_000);
|
||||
expect(stagingMixHeadline(result)).toEqual([
|
||||
"completed hands-off / started: 2 / 5",
|
||||
"Retry banner shown: 3 tasks",
|
||||
"Retry worked / Retry clicked: 1 / 2",
|
||||
]);
|
||||
expect(result.bannerShown.find(entry => entry.taskId === "t2")).toMatchObject({ completedAnyway: true, errorCode: "controller_lease_lost", shownMs: 30_000 });
|
||||
expect(result.invariant.violationCount).toBe(2);
|
||||
});
|
||||
|
||||
it("reads the assigned-work invariant once as counts, and records a failed read", async () => {
|
||||
const report = { checked: 4, openTasks: 3, violations: { nothing_pending: 1 }, violationCount: 1, agentsWithStalledSlots: [{ agentId: "a" }] };
|
||||
expect(await readAssignedWorkInvariant({ get: async <T>() => report as T }, "c1"))
|
||||
.toEqual({ violationCount: 1, violations: { nothing_pending: 1 }, agentsWithStalledSlots: 1, openTasks: 3 });
|
||||
expect(await readAssignedWorkInvariant({ get: async () => { throw new Error("HTTP 404"); } }, "c1")).toMatchObject({ violationCount: null, error: "Error: HTTP 404" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("staging-mix completion comments", () => {
|
||||
const nonce = "b3d70e77";
|
||||
const base = (comments: Array<{ body: string; runId: string | null }>): OracleArtifacts => ({ issueStatus: "done", documents: { ledger: ledger(nonce, 4) },
|
||||
agentComments: comments.map(comment => comment.body), files: {}, commands: {}, workspacePath: null,
|
||||
stagingMix: { followUp: null, followUpErrors: [], retries: [], comments } });
|
||||
const check = (artifacts: OracleArtifacts, name: string) => gradeStagingMix(nonce, task(), artifacts).checks.find(entry => entry.name === name)!;
|
||||
|
||||
it("counts only the DONE comment itself, not a closing summary that quotes it", () => {
|
||||
const comments = [
|
||||
{ body: `CHA-1 complete. Saved the ledger and posted \`${doneMarker(nonce)}\`.`, runId: "run-b" },
|
||||
{ body: "Prepared CHA-1 ledger for this response.", runId: "run-b" },
|
||||
{ body: doneMarker(nonce), runId: "run-b" },
|
||||
];
|
||||
expect(check(base(comments), "done_comment_once")).toMatchObject({ ok: true, detail: "comments=1" });
|
||||
expect(check(base(comments), "completion_once").ok).toBe(true);
|
||||
expect(gradeStagingMix(nonce, task(), base(comments)).verdict).toBe("pass");
|
||||
expect(check(base([...comments, { body: ` \`${doneMarker(nonce)}\`. `, runId: "run-c" }]), "done_comment_once")).toMatchObject({ ok: false, detail: "comments=2" });
|
||||
});
|
||||
|
||||
it("flags a completion summary repeated by a different run as a duplicate completion", () => {
|
||||
const comments = [
|
||||
{ body: `CHA-1 complete. Posted exactly one comment ${doneMarker(nonce)} and marked the task done.`, runId: "413ddcf2-ffce" },
|
||||
{ body: `CHA-1 complete. Posted \`${doneMarker(nonce)}\`.`, runId: "13de31be-fb13" },
|
||||
{ body: doneMarker(nonce), runId: "13de31be-fb13" },
|
||||
];
|
||||
expect(check(base(comments), "done_comment_once").ok).toBe(true);
|
||||
expect(check(base(comments), "completion_once")).toMatchObject({ ok: false, detail: "duplicate completion: runs 413ddcf2, 13de31be each reported completion" });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,152 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import type { ChaosApi } from "./api.js";
|
||||
import { bannerShownMs, noteRetryBanner } from "./staging-mix.js";
|
||||
import type { InterruptionSample, TaskPlan, TaskRecord, TaskSample } from "./types.js";
|
||||
|
||||
type Json = Record<string, any>;
|
||||
|
||||
export interface IssueView { status: string; canRetry: boolean; retryWait: boolean; observedRunId: string | null; interruption: InterruptionSample | null }
|
||||
|
||||
export function shouldClickRetryAfterSettle(view: Pick<IssueView, "status" | "canRetry" | "retryWait">, priorClicks: number): boolean {
|
||||
return view.status !== "done" && view.canRetry && !view.retryWait && priorClicks === 0;
|
||||
}
|
||||
|
||||
export function issueView(issue: Json | null): IssueView {
|
||||
const interruption = issue?.interruption ?? null;
|
||||
return {
|
||||
status: String(issue?.status ?? "unreadable"),
|
||||
canRetry: interruption?.canRetry === true,
|
||||
retryWait: Boolean(interruption?.retryWait),
|
||||
observedRunId: typeof interruption?.observedRunId === "string" ? interruption.observedRunId : null,
|
||||
interruption: interruption ? { state: String(interruption.state), canRetry: interruption.canRetry === true, retryWait: Boolean(interruption.retryWait),
|
||||
workspaceFailure: null, observedRunId: typeof interruption.observedRunId === "string" ? interruption.observedRunId : null,
|
||||
userAction: typeof interruption.userAction === "string" ? interruption.userAction : null } : null,
|
||||
};
|
||||
}
|
||||
|
||||
export interface RetryStageObserver {
|
||||
plan: TaskPlan;
|
||||
record: TaskRecord;
|
||||
done: boolean;
|
||||
poll(now?: number): Promise<void>;
|
||||
retryAfterSettle(sample: TaskSample, now: number): Promise<void>;
|
||||
}
|
||||
|
||||
export interface RetryStageInput {
|
||||
observers: readonly RetryStageObserver[];
|
||||
api: Pick<ChaosApi, "get">;
|
||||
settleUntil: number;
|
||||
settleMs: number;
|
||||
cancelled: () => boolean;
|
||||
note: (kind: string, data: Record<string, unknown>) => void;
|
||||
pollMs?: number;
|
||||
now?: () => number;
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
}
|
||||
|
||||
export async function retryAfterSettleStage(input: RetryStageInput): Promise<{ clicked: string[] }> {
|
||||
const now = input.now ?? Date.now;
|
||||
const sleep = input.sleep ?? (ms => delay(ms));
|
||||
const pollMs = input.pollMs ?? 60_000;
|
||||
const tracked = input.observers.filter(observer => observer.record.issueId && observer.plan.stagingMix);
|
||||
const views = new Map<RetryStageObserver, IssueView>();
|
||||
const read = async (observers: readonly RetryStageObserver[]) => {
|
||||
for (const observer of observers) {
|
||||
const issue = await input.api.get<Json>(`/api/issues/${observer.record.issueId}`, 10_000).catch(() => null);
|
||||
const view = issueView(issue);
|
||||
views.set(observer, view);
|
||||
noteRetryBanner(observer.record.stagingMix ??= {}, view.status !== "done" && view.canRetry, now(), null);
|
||||
}
|
||||
};
|
||||
input.note("staging_mix_settling", { settleUntil: new Date(input.settleUntil).toISOString(), tasks: tracked.length });
|
||||
for (;;) {
|
||||
await read(tracked.filter(observer => views.get(observer)?.status !== "done"));
|
||||
if (input.cancelled() || now() >= input.settleUntil || tracked.every(observer => views.get(observer)?.status === "done")) break;
|
||||
await sleep(Math.min(pollMs, Math.max(1, input.settleUntil - now())));
|
||||
}
|
||||
const clicked: RetryStageObserver[] = [];
|
||||
for (const observer of tracked) {
|
||||
const view = views.get(observer);
|
||||
if (!view || input.cancelled() || !shouldClickRetryAfterSettle(view, observer.record.retry ? 1 : 0)) continue;
|
||||
const at = now();
|
||||
await observer.retryAfterSettle({ at, reachable: true, serverReady: true, issueStatus: view.status, interruption: view.interruption }, at);
|
||||
(observer.record.stagingMix ??= {}).retryAfterSettle = { at, accepted: observer.record.retry?.accepted ?? null, issueStatusAtClick: view.status };
|
||||
input.note("staging_mix_retry_after_settle", { taskId: observer.plan.taskId, accepted: observer.record.retry?.accepted ?? null });
|
||||
clicked.push(observer);
|
||||
}
|
||||
const until = now() + input.settleMs;
|
||||
while (clicked.some(observer => !observer.done) && !input.cancelled() && now() < until) {
|
||||
for (const observer of clicked) if (!observer.done) await observer.poll(now());
|
||||
await sleep(Math.min(pollMs, 15_000));
|
||||
}
|
||||
return { clicked: clicked.map(observer => observer.plan.taskId) };
|
||||
}
|
||||
|
||||
export interface ResultTask { taskId: string; issueId: string | null; record: TaskRecord; pass: boolean; finalStatus: string | null }
|
||||
|
||||
export interface StagingMixResult {
|
||||
started: number;
|
||||
completedHandsOff: number;
|
||||
bannerShown: Array<{ taskId: string; firstShownAt: string; shownMs: number; errorCode: string | null; completedAnyway: boolean }>;
|
||||
retryClicked: number;
|
||||
retryWorked: number;
|
||||
invariant: { violationCount: number | null; violations: Record<string, number> | null; agentsWithStalledSlots: number | null; openTasks: number | null; error?: string };
|
||||
}
|
||||
|
||||
export function summarizeStagingMix(tasks: readonly ResultTask[], extraClicks: ReadonlyMap<string, boolean | null>, invariant: StagingMixResult["invariant"], now = Date.now()): StagingMixResult {
|
||||
const clicked = (task: ResultTask) => Boolean(task.record.retry || task.record.priorRetries?.length || (task.issueId && extraClicks.has(task.issueId)));
|
||||
const clickedTasks = tasks.filter(clicked);
|
||||
return {
|
||||
started: tasks.length,
|
||||
completedHandsOff: tasks.filter(task => task.pass && !clicked(task)).length,
|
||||
bannerShown: tasks.flatMap(task => task.record.stagingMix?.retryBanner ? [{ taskId: task.taskId, firstShownAt: new Date(task.record.stagingMix.retryBanner.firstShownAt).toISOString(),
|
||||
shownMs: bannerShownMs(task.record.stagingMix.retryBanner, now), errorCode: task.record.stagingMix.retryBanner.errorCode,
|
||||
completedAnyway: task.finalStatus === "done" && !clicked(task) }] : []),
|
||||
retryClicked: clickedTasks.length,
|
||||
retryWorked: clickedTasks.filter(task => task.finalStatus === "done").length,
|
||||
invariant,
|
||||
};
|
||||
}
|
||||
|
||||
export function stagingMixHeadline(result: StagingMixResult): string[] {
|
||||
return [
|
||||
`completed hands-off / started: ${result.completedHandsOff} / ${result.started}`,
|
||||
`Retry banner shown: ${result.bannerShown.length} tasks`,
|
||||
`Retry worked / Retry clicked: ${result.retryWorked} / ${result.retryClicked}`,
|
||||
];
|
||||
}
|
||||
|
||||
export async function readAssignedWorkInvariant(api: Pick<ChaosApi, "get">, companyId: string): Promise<StagingMixResult["invariant"]> {
|
||||
try {
|
||||
const report = await api.get<Json>(`/api/companies/${companyId}/assigned-work-invariant`, 30_000);
|
||||
return { violationCount: typeof report.violationCount === "number" ? report.violationCount : null, violations: report.violations ?? null,
|
||||
agentsWithStalledSlots: Array.isArray(report.agentsWithStalledSlots) ? report.agentsWithStalledSlots.length : null,
|
||||
openTasks: typeof report.openTasks === "number" ? report.openTasks : null };
|
||||
} catch (error) {
|
||||
return { violationCount: null, violations: null, agentsWithStalledSlots: null, openTasks: null, error: String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
export async function writeStagingMixResults(input: { waveDir: string; api: Pick<ChaosApi, "get">; companyId: string;
|
||||
tasks: ReadonlyArray<{ plan: TaskPlan; record: TaskRecord }>; passed: ReadonlyMap<string, boolean>;
|
||||
timeline: ReadonlyArray<{ kind: string; [key: string]: unknown }> }): Promise<StagingMixResult> {
|
||||
const extraClicks = new Map<string, boolean | null>(input.timeline.filter(entry => entry.kind === "release_swap_retry_clicked" && typeof entry.issueId === "string")
|
||||
.map(entry => [String(entry.issueId), entry.accepted === true ? true : entry.accepted === false ? false : null]));
|
||||
const tasks: ResultTask[] = [];
|
||||
for (const task of input.tasks) {
|
||||
const issue = task.record.issueId ? await input.api.get<Json>(`/api/issues/${task.record.issueId}`, 10_000).catch(() => null) : null;
|
||||
tasks.push({ taskId: task.plan.taskId, issueId: task.record.issueId, record: task.record, pass: input.passed.get(task.plan.taskId) === true,
|
||||
finalStatus: issue?.status ?? task.record.finalStatus });
|
||||
}
|
||||
const result = summarizeStagingMix(tasks, extraClicks, await readAssignedWorkInvariant(input.api, input.companyId));
|
||||
await writeFile(path.join(input.waveDir, "staging-mix-results.json"), `${JSON.stringify({ schema: "paperclip.restart-chaos.staging-mix/v1",
|
||||
headline: stagingMixHeadline(result), ...result }, null, 2)}\n`, { mode: 0o600 });
|
||||
const invariant = result.invariant.violationCount === null ? `unreadable${result.invariant.error ? ` (${result.invariant.error})` : ""}`
|
||||
: `${result.invariant.violationCount} violations, ${result.invariant.agentsWithStalledSlots} agents with stalled slots, ${result.invariant.openTasks} open tasks`;
|
||||
await writeFile(path.join(input.waveDir, "staging-mix-summary.md"), [...stagingMixHeadline(result), `Assigned-work invariant: ${invariant}`, "",
|
||||
...result.bannerShown.map(entry => `- ${entry.taskId}: Retry banner from ${entry.firstShownAt} for ${Math.round(entry.shownMs / 1000)} s`
|
||||
+ `${entry.errorCode ? ` (${entry.errorCode})` : ""}${entry.completedAnyway ? ", completed without a click" : ""}`), ""].join("\n"), { mode: 0o600 });
|
||||
return result;
|
||||
}
|
||||
@@ -3,7 +3,9 @@ import { readFileSync } from "node:fs";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { ChaosApi } from "./api.js";
|
||||
import { agentRunCaps, type StagingMixOptions, type StagingMixSlot } from "./staging-mix.js";
|
||||
import { retryAfterSettleStage, type RetryStageObserver } from "./staging-mix-retry.js";
|
||||
import type { AdapterId, TaskPlan, WaveOptions } from "./types.js";
|
||||
|
||||
export const RESOURCE_SAMPLE_MS = 30_000;
|
||||
@@ -128,6 +130,24 @@ export class StagingMixRun {
|
||||
return problems;
|
||||
}
|
||||
|
||||
report(started: number): Error | null {
|
||||
const problems = this.shortfall(started);
|
||||
if (problems.length) this.note("staging_mix_shortfall", { problems, unfired: this.unfiredSlots() });
|
||||
console.info(`Staging mix: ${started} tasks started; ${this.unlaunched} planned tasks never launched (excluded from tasks started); `
|
||||
+ `fault slots fired ${this.slotCoverage.map(entry => `${Math.round(entry.atMs / 60_000)}m:${entry.kind}×${entry.exposed} tasks`).join(", ") || "none"} of ${this.slots.length}`);
|
||||
return problems.length ? new Error(`Staging mix tested less than planned: ${problems.join("; ")}`) : null;
|
||||
}
|
||||
|
||||
settleUntil(now = Date.now()): number {
|
||||
const last = this.slots.at(-1);
|
||||
return Math.max(now, last ? this.startedAt + last.atMs + last.holdMs : now) + this.mix.settleMinutes * 60_000;
|
||||
}
|
||||
|
||||
async retryAfterSettle(observers: readonly RetryStageObserver[], api: Pick<ChaosApi, "get">, cancelled: () => boolean): Promise<void> {
|
||||
if (this.mix.retryDuringWave) return;
|
||||
await retryAfterSettleStage({ observers, api, settleUntil: this.settleUntil(), settleMs: this.mix.settleMinutes * 60_000, cancelled, note: this.note });
|
||||
}
|
||||
|
||||
mayLaunch(now: number, next: TaskPlan): boolean {
|
||||
if (this.ended || !next.stagingMix || now < this.startedAt + next.stagingMix.arriveAtMs) return false;
|
||||
const latest = this.samples[this.samples.length - 1];
|
||||
|
||||
@@ -264,6 +264,7 @@ describe("staging-mix board step", () => {
|
||||
const failed: TaskSample = { at: 0, reachable: true, serverReady: true, issueStatus: "blocked",
|
||||
interruption: { state: "failed", canRetry: true, retryWait: false, workspaceFailure: null, observedRunId: "r1", userAction: null },
|
||||
runs: [{ ...running.runs![0]!, status: "failed" }] };
|
||||
const stepIn = (rec: TaskRecord, sample: TaskSample, now: number, faultInProgress: boolean, plan: StagingMixTask) => step(rec, sample, now, faultInProgress, plan);
|
||||
const step = async (rec: TaskRecord, sample: TaskSample, now: number, faultInProgress: boolean, plan = task({ followUpAfterMs: 60_000 }),
|
||||
reply: () => unknown = () => ({ id: "c1" })) => {
|
||||
const posted: string[] = [], retried: number[] = [], notes: string[] = [];
|
||||
@@ -309,8 +310,18 @@ describe("staging-mix board step", () => {
|
||||
expect((await step(rec, running, Math.round(4 * ROUND_SECONDS * 1_000 * 0.85), false, plan)).posted.length).toBe(1);
|
||||
});
|
||||
|
||||
it("waits for an overlap to click a Retry, and holds the observer's own click until the hold runs out", async () => {
|
||||
it("never clicks Retry during the wave by default, but records the banner", async () => {
|
||||
const rec = record();
|
||||
expect((await step(rec, failed, 1_000, true)).retried).toEqual([]);
|
||||
expect((await step(rec, failed, 9_000, true)).retried).toEqual([]);
|
||||
expect(rec.stagingMix!.retryBanner).toMatchObject({ firstShownAt: 1_000, lastShownAt: 9_000, visibleSince: 1_000 });
|
||||
await step(rec, running, 20_000, false);
|
||||
expect(rec.stagingMix!.retryBanner).toMatchObject({ shownMs: 19_000, visibleSince: null });
|
||||
});
|
||||
|
||||
it("with --mix-retry-during-wave, waits for an overlap to click a Retry, and holds the observer's own click until the hold runs out", async () => {
|
||||
const rec = record();
|
||||
const step = (r: TaskRecord, sample: TaskSample, now: number, overlap: boolean) => stepIn(r, sample, now, overlap, task({ followUpAfterMs: 60_000, retryDuringWave: true }));
|
||||
expect((await step(rec, failed, 1_000, false)).retried).toEqual([]);
|
||||
expect(holdsRetry(rec, 600_000, 2_000)).toBe(true);
|
||||
expect(holdsRetry(rec, 600_000, 1_000 + 600_000)).toBe(false);
|
||||
|
||||
@@ -30,6 +30,8 @@ export interface StagingMixOptions {
|
||||
minFreeMb: number;
|
||||
arrivalCutoffMinutes: number;
|
||||
arrivalEndMs: number;
|
||||
retryDuringWave: boolean;
|
||||
settleMinutes: number;
|
||||
tasks?: number;
|
||||
}
|
||||
|
||||
@@ -42,6 +44,7 @@ export interface StagingMixTask {
|
||||
followUpAfterMs: number | null;
|
||||
arriveAtMs: number;
|
||||
holdMs: number;
|
||||
retryDuringWave?: boolean;
|
||||
}
|
||||
|
||||
export interface StagingMixRecord {
|
||||
@@ -49,12 +52,17 @@ export interface StagingMixRecord {
|
||||
followUp?: { at: number; commentId: string | null; duringRun: boolean; inOverlap: boolean; error: string | null };
|
||||
retrySeenAt?: number | null;
|
||||
retries?: Array<{ at: number; inOverlap: boolean }>;
|
||||
retryBanner?: RetryBanner;
|
||||
retryAfterSettle?: { at: number; accepted: boolean | null; issueStatusAtClick: string | null };
|
||||
}
|
||||
|
||||
export interface RetryBanner { firstShownAt: number; lastShownAt: number; shownMs: number; visibleSince: number | null; errorCode: string | null }
|
||||
|
||||
export interface StagingMixArtifacts {
|
||||
followUp: StagingMixRecord["followUp"] | null;
|
||||
followUpErrors: string[];
|
||||
retries: NonNullable<StagingMixRecord["retries"]>;
|
||||
comments?: Array<{ body: string; runId: string | null }>;
|
||||
}
|
||||
|
||||
export const STAGING_MIX_ARG_OPTIONS = {
|
||||
@@ -71,6 +79,8 @@ export const STAGING_MIX_ARG_OPTIONS = {
|
||||
"mix-run-cap": { type: "string" },
|
||||
"mix-tasks": { type: "string" },
|
||||
"mix-arrival-cutoff-minutes": { type: "string" },
|
||||
"mix-retry-during-wave": { type: "boolean" },
|
||||
"mix-settle-minutes": { type: "string" },
|
||||
} as const;
|
||||
|
||||
type MixArgs = Partial<Record<keyof typeof STAGING_MIX_ARG_OPTIONS, string | boolean | undefined>> & Record<string, unknown>;
|
||||
@@ -141,6 +151,8 @@ export function applyStagingMix(options: WaveOptions, values: MixArgs, environme
|
||||
minFreeMb: whole(environment.RESTART_CHAOS_MIX_MIN_FREE_MB, "RESTART_CHAOS_MIX_MIN_FREE_MB", DEFAULT_MIN_FREE_MB, 0, 1_048_576),
|
||||
arrivalCutoffMinutes,
|
||||
arrivalEndMs: arrivalEndMs({ minutes, faultSpacingMinutes, arrivalCutoffMinutes }, options),
|
||||
retryDuringWave: values["mix-retry-during-wave"] === true,
|
||||
settleMinutes: whole(values["mix-settle-minutes"], "mix-settle-minutes", 15, 0, 120),
|
||||
...(values["mix-tasks"] !== undefined ? { tasks: whole(values["mix-tasks"], "mix-tasks", 1, 1, 2_000) } : {}),
|
||||
};
|
||||
if (mix.tasks === undefined && mix.arrivalEndMs < (mix.runs + mix.queue) * BURST_GAP_MS) {
|
||||
@@ -269,7 +281,8 @@ export function planStagingMixTasks(options: WaveOptions, rng = createRng((optio
|
||||
holdMs: 0,
|
||||
followUpAfterMs: null,
|
||||
nonce: rng.hex(8),
|
||||
stagingMix: { turnMinutes, rounds, subtasks, followUpAfterMs, arriveAtMs, holdMs: mix.holdMinutes * 60_000 },
|
||||
stagingMix: { turnMinutes, rounds, subtasks, followUpAfterMs, arriveAtMs, holdMs: mix.holdMinutes * 60_000,
|
||||
...(mix.retryDuringWave ? { retryDuringWave: true } : {}) },
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -354,6 +367,10 @@ export function stagingMixDescription(nonce: string, task: StagingMixTask, agent
|
||||
|
||||
export const followUpRequestId = (issueId: string, nonce: string) => stableRequestId(`followup-${issueId}-${nonce}`);
|
||||
|
||||
export function isDoneComment(body: string, nonce: string): boolean {
|
||||
return body.trim().replace(/\.$/, "").replace(/^`+|`+$/g, "").trim() === doneMarker(nonce);
|
||||
}
|
||||
|
||||
export function followUpComment(nonce: string): string {
|
||||
return `Additional requirement: include the line \`${followUpMarker(nonce)}\` on its own line in this task's \`ledger\` document. Keep going with your rounds; do not restart any.`;
|
||||
}
|
||||
@@ -387,7 +404,9 @@ export function verifyLedger(nonce: string, rounds: number, text: string | null
|
||||
export function gradeStagingMix(nonce: string, task: StagingMixTask, artifacts: OracleArtifacts): OracleResult {
|
||||
const ledger = artifacts.documents.ledger ?? null;
|
||||
const own = verifyLedger(nonce, task.rounds, ledger);
|
||||
const doneComments = artifacts.agentComments.filter(body => body.includes(doneMarker(nonce))).length;
|
||||
const comments = artifacts.stagingMix?.comments ?? artifacts.agentComments.map(body => ({ body, runId: null }));
|
||||
const doneComments = comments.filter(comment => isDoneComment(comment.body, nonce)).length;
|
||||
const completionRuns = [...new Set(comments.filter(comment => comment.body.includes(doneMarker(nonce)) && comment.runId).map(comment => comment.runId!))];
|
||||
const followUp = artifacts.stagingMix?.followUp ?? null;
|
||||
const children = (artifacts.children ?? []).filter(child => (child.title ?? "").includes(nonce));
|
||||
const checks: OracleResult["checks"] = [
|
||||
@@ -395,6 +414,8 @@ export function gradeStagingMix(nonce: string, task: StagingMixTask, artifacts:
|
||||
{ name: "ledger_rounds_verified", ok: own.verified, detail: own.detail },
|
||||
{ name: "rounds_once", ok: own.duplicates === 0, detail: `duplicates=${own.duplicates}` },
|
||||
{ name: "done_comment_once", ok: doneComments === 1, detail: `comments=${doneComments}` },
|
||||
{ name: "completion_once", ok: completionRuns.length <= 1,
|
||||
detail: completionRuns.length > 1 ? `duplicate completion: runs ${completionRuns.map(id => id.slice(0, 8)).join(", ")} each reported completion` : `runs=${completionRuns.length}` },
|
||||
];
|
||||
if (task.followUpAfterMs !== null) {
|
||||
const errors = artifacts.stagingMix?.followUpErrors ?? [];
|
||||
@@ -502,17 +523,46 @@ export async function stagingMixStep(input: StagingMixStepInput): Promise<void>
|
||||
input.note("staging_mix_follow_up", { ...followUp });
|
||||
}
|
||||
}
|
||||
noteRetryBanner(state, retryShown(sample), now, latestFailureCode(sample));
|
||||
if (!retryOffered(sample) || !retryDue(record)) {
|
||||
if (!retryOffered(sample)) state.retrySeenAt = null;
|
||||
return;
|
||||
}
|
||||
state.retrySeenAt ??= now;
|
||||
if (!overlap) return;
|
||||
if (!overlap || !task.retryDuringWave) return;
|
||||
(state.retries ??= []).push({ at: now, inOverlap: true });
|
||||
input.note("staging_mix_retry_in_overlap", { waitedMs: now - state.retrySeenAt });
|
||||
await input.retry(sample, now);
|
||||
}
|
||||
|
||||
export function stagingMixArtifacts(record: TaskRecord): StagingMixArtifacts {
|
||||
return { followUp: record.stagingMix?.followUp ?? null, followUpErrors: record.stagingMix?.followUpErrors ?? [], retries: record.stagingMix?.retries ?? [] };
|
||||
export function retryShown(sample: Pick<TaskSample, "reachable" | "issueStatus" | "interruption">): boolean {
|
||||
return sample.reachable && sample.issueStatus !== "done" && Boolean(sample.interruption?.canRetry);
|
||||
}
|
||||
|
||||
export function latestFailureCode(sample: Pick<TaskSample, "runs">): string | null {
|
||||
const failed = (sample.runs ?? []).filter(run => run.status === "failed" || run.status === "timed_out" || run.errorCode)
|
||||
.sort((a, b) => Date.parse(b.createdAt ?? "") - Date.parse(a.createdAt ?? ""));
|
||||
return failed[0]?.errorCode ?? failed[0]?.failureCause ?? null;
|
||||
}
|
||||
|
||||
export function noteRetryBanner(state: StagingMixRecord, shown: boolean, now: number, errorCode: string | null): void {
|
||||
const banner = state.retryBanner;
|
||||
if (!shown) {
|
||||
if (banner?.visibleSince != null) { banner.shownMs += now - banner.visibleSince; banner.visibleSince = null; }
|
||||
return;
|
||||
}
|
||||
if (!banner) { state.retryBanner = { firstShownAt: now, lastShownAt: now, shownMs: 0, visibleSince: now, errorCode }; return; }
|
||||
banner.lastShownAt = now;
|
||||
banner.visibleSince ??= now;
|
||||
banner.errorCode = errorCode ?? banner.errorCode;
|
||||
}
|
||||
|
||||
export function bannerShownMs(banner: RetryBanner, now: number): number {
|
||||
return banner.shownMs + (banner.visibleSince === null ? 0 : now - banner.visibleSince);
|
||||
}
|
||||
|
||||
export async function collectStagingMixArtifacts(api: Pick<ChaosApi, "get">, issueId: string, record: TaskRecord): Promise<StagingMixArtifacts> {
|
||||
const rows = await api.get<Array<{ body?: string; authorAgentId?: string | null; createdByRunId?: string | null }>>(`/api/issues/${issueId}/comments`).catch(() => null);
|
||||
return { followUp: record.stagingMix?.followUp ?? null, followUpErrors: record.stagingMix?.followUpErrors ?? [], retries: record.stagingMix?.retries ?? [],
|
||||
...(rows ? { comments: rows.filter(row => row.authorAgentId).map(row => ({ body: String(row.body ?? ""), runId: row.createdByRunId ?? null })) } : {}) };
|
||||
}
|
||||
Reference in new issue
Block a user