diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 50878cbcca7..d6cc3949994 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -303,7 +303,13 @@ inside or outside a release window (from the old build's last answer to 60 s aft on no answer, a server error, or 401, 403, 408 or 429; a deliberate 400, 404, 409 or 422 answer is not a failure. Runs come from the company run list, or from every task's run list when that list is full, so no task's early runs are lost. A queued or running run with no activity for 5 minutes is not a live owner, the same window the drain uses. The drain also waits for the 90 s quiet period -(`--hosted-quiet-seconds`), and the grade reads runs and task views again immediately before grading. +(`--hosted-quiet-seconds`), and the grade reads runs and task views again immediately before grading. The drain cap +counts from the later of the last release and the end of arrivals, so a run that stops launching at `--hosted-max-hours` +long after its last release still drains for `--hosted-drain-minutes`. Run activity is aged from the moment the grade +read the runs, not from after the task view reads. A task that is still open, has no failed run, and whose every live run is still queued +behind its agent's full `maxConcurrentRuns` cap, with the queue-wait proof finding no idle slot and no newer +assignment run started ahead of it, is not graded: `SUMMARY.md` names it on a `Not graded` line and +`hosted-release-swap.json` lists it under `notGraded`. The runs holding the slots are still graded. **Gate D verdict** (`hosted-gate.ts`). A run exits 0 only when every check in the `Gate D` block of `SUMMARY.md` holds: every started task completed hands-off; every task the harness launched was created; every started task passes the local waves' diff --git a/tests/restart-chaos/SCENARIOS.md b/tests/restart-chaos/SCENARIOS.md index 7787077e21b..d8bc65b36c5 100644 --- a/tests/restart-chaos/SCENARIOS.md +++ b/tests/restart-chaos/SCENARIOS.md @@ -643,7 +643,9 @@ A's SIGTERM to the next event of each run that was live across it and did not fa every grade, every failed run with its error code, message and last error events, each swap's run samples and probes, and the duplicates: two runs of one task executing at once, two Runner processes for one run, repeated identical agent comments and repeated subtask titles. The product's per-turn notice "Prepared for this -response." is not a duplicated comment; its repeats are counted on the same line as information only. The wave fails +response." is not a duplicated comment, and neither is the loop-guard notice "Paperclip stopped a tool call that kept +failing the same way. The task continues in a fresh session." when each copy came from a different run; their repeats +are counted on the same line as information only. Two copies of the loop-guard notice from one run are a duplicate. The wave fails unless every task passed. **Swap moments.** Each swap has five moments, in the timeline as `swap_incoming_spawned`, `swap_incoming_ready`, diff --git a/tests/restart-chaos/hosted-cutoff.test.ts b/tests/restart-chaos/hosted-cutoff.test.ts new file mode 100644 index 00000000000..d5fa276c33c --- /dev/null +++ b/tests/restart-chaos/hosted-cutoff.test.ts @@ -0,0 +1,174 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { hostedQueueWait, queuedBehindSlotCap, slotQueuedRuns } from "./hosted-gate.js"; +import { drainHosted, gradeHosted } from "./hosted-grade.js"; +import { duplicateAgentComments, runRow } from "./hosted-readers.js"; +import { continuity } from "./hosted-release.js"; +import type { HostedTaskRecord } from "./hosted-workload.js"; +import { isNoticeRepeat, LOOP_GUARD_NOTICE } from "./release-swap-grade.js"; +import { roundsFor } from "./staging-mix.js"; + +type Json = Record; +const dirs: string[] = []; +afterEach(async () => { for (const dir of dirs.splice(0)) await rm(dir, { recursive: true, force: true }); }); + +const T0 = Date.parse("2026-11-22T05:00:00.000Z"); +const MIN = 60_000; +const iso = (at: number) => new Date(at).toISOString(); +const task = { turnMinutes: 15, rounds: roundsFor(15), subtasks: 2, followUpAfterMs: null, arriveAtMs: 0, holdMs: 0 }; + +function planned(issueId: string, identifier: string, createdAt: number): HostedTaskRecord { + return { plan: { index: 0, adapter: "opencode", nonce: identifier, task }, issueId, identifier, createdAt, createError: null, status: "in_progress", + lastLiveAt: null, followUp: null, retryClicks: [], retriedRuns: [], doneAt: null }; +} + +const apiRun = (id: string, issueId: string, createdAt: number, startedAt: number | null, extra: Json = {}): Json => ({ id, agentId: "a1", issueId, + status: startedAt === null ? "queued" : "running", invocationSource: "assignment", createdAt: iso(createdAt), startedAt: startedAt === null ? null : iso(startedAt), + finishedAt: null, updatedAt: iso(startedAt ?? createdAt), ...extra }); + +describe("end-of-wave drain after a run that stopped at the max-hours cap", () => { + it("drains from the end of arrivals, not from a release fifteen hours earlier", async () => { + let clock = T0; + const finishAt = T0 + 20 * MIN; + const api = { get: async () => { + clock += MIN; + const done = clock >= finishAt; + return [{ id: "r1", agentId: "a1", issueId: "i1", status: done ? "succeeded" : "running", createdAt: iso(T0 - 5 * MIN), startedAt: iso(T0 - 5 * MIN), + finishedAt: done ? iso(finishAt) : null, updatedAt: iso(Math.min(clock, finishAt)) }] as T; + }, post: async () => ({}) as T }; + const events: Json[] = []; + const drained = await drainHosted({ api, companyId: "c1", agentIds: ["a1"], lastReleaseAt: T0 - 15 * 60 * MIN, settleMs: 15 * MIN, drainMs: 57 * MIN, + cancelled: () => false, record: (kind, data) => events.push({ kind, ...data }), pollMs: 1, now: () => clock, quietMs: 90_000 }); + expect(drained.reason).toBe("quiet"); + expect(clock).toBeGreaterThanOrEqual(finishAt + 90_000); + expect(events[0]).toMatchObject({ kind: "hosted_settling", drainFrom: iso(T0), capAt: iso(T0 + 57 * MIN) }); + }); + + it("still stops at a bounded cap when a run never goes quiet", async () => { + let clock = T0; + const api = { get: async () => { + clock += MIN; + return [{ id: "r1", agentId: "a1", issueId: "i1", status: "running", createdAt: iso(T0), startedAt: iso(T0), finishedAt: null, updatedAt: iso(clock) }] as T; + }, post: async () => ({}) as T }; + const drained = await drainHosted({ api, companyId: "c1", agentIds: ["a1"], lastReleaseAt: T0 - 15 * 60 * MIN, settleMs: 15 * MIN, drainMs: 30 * MIN, + cancelled: () => false, record: () => {}, pollMs: 1, now: () => clock, quietMs: 90_000 }); + expect(drained).toEqual({ reason: "cap", progressing: 1 }); + expect(clock).toBeGreaterThanOrEqual(T0 + 30 * MIN); + expect(clock).toBeLessThan(T0 + 32 * MIN); + }); +}); + +describe("runs queued behind a full agent slot cap", () => { + const agents = [{ id: "a1", runtimeConfig: { heartbeat: { maxConcurrentRuns: 2 } } }]; + const rows = (list: Json[]) => list.map(runRow); + const queued = (list: Json[], at: number) => { + const runs = rows(list); + return slotQueuedRuns(runs, at, hostedQueueWait(["a1"], agents, runs)); + }; + + it("counts a never-started run as slot-queued only while the running runs fill the cap with no idle slot since it was created", () => { + const full = [apiRun("h1", "i1", T0, T0), apiRun("h2", "i2", T0 + MIN, T0 + MIN), apiRun("q1", "i3", T0 + 2 * MIN, null)]; + expect([...queued(full, T0 + 8 * MIN)]).toEqual(["q1"]); + const oneRunning = [apiRun("h1", "i1", T0, T0), apiRun("h2", "i2", T0 + MIN, T0 + MIN, { status: "succeeded", finishedAt: iso(T0 + 3 * MIN) }), + apiRun("q1", "i3", T0 + 2 * MIN, null)]; + expect([...queued(oneRunning, T0 + 8 * MIN)]).toEqual([]); + const refilledLate = [...oneRunning, apiRun("h3", "i4", T0 + 6 * MIN, T0 + 6 * MIN, { invocationSource: "on_demand" })]; + expect([...queued(refilledLate, T0 + 8 * MIN)]).toEqual([]); + const overtaken = [apiRun("h1", "i1", T0, T0), apiRun("h2", "i2", T0 + MIN, T0 + MIN, { status: "succeeded", finishedAt: iso(T0 + 3 * MIN) }), + apiRun("q1", "i3", T0 + 2 * MIN, null), apiRun("h3", "i4", T0 + 2.5 * MIN, T0 + 3 * MIN)]; + expect([...queued(overtaken, T0 + 8 * MIN)]).toEqual([]); + expect([...slotQueuedRuns(rows(full), T0 + 8 * MIN, hostedQueueWait(["a1"], [{ id: "a1", runtimeConfig: null }], rows(full)))]).toEqual([]); + }); + + it("leaves a task to the grade unless every live run of it is slot-queued and it is still open", () => { + const runs = rows([apiRun("q1", "i3", T0, null), apiRun("r9", "i3", T0, T0)]); + expect(queuedBehindSlotCap("todo", runs.slice(0, 1), new Set(["q1"]))).toBe(true); + expect(queuedBehindSlotCap("todo", runs, new Set(["q1"]))).toBe(false); + expect(queuedBehindSlotCap("blocked", runs.slice(0, 1), new Set(["q1"]))).toBe(false); + expect(queuedBehindSlotCap("todo", [], new Set(["q1"]))).toBe(false); + const failedFirst = rows([apiRun("q1", "i3", T0, null), apiRun("f1", "i3", T0 - MIN, T0 - MIN, { status: "failed", finishedAt: iso(T0) })]); + expect(queuedBehindSlotCap("todo", failedFirst, new Set(["q1"]))).toBe(false); + }); +}); + +describe("hosted grade at the cutoff", () => { + const grade = async (input: { runs: Json[] | ((clock: number) => Json[]); issues: Json[]; tasks: HostedTaskRecord[]; cap: number; start: number; viewMs: number }) => { + let clock = input.start; + const views = new Map(input.issues.map(issue => [`/api/issues/${issue.id}`, issue])); + const api = { + get: async (route: string): Promise => { + if (route.includes("/heartbeat-runs?")) { clock += 1_000; return (typeof input.runs === "function" ? input.runs(clock) : input.runs) as T; } + if (route.includes("/issues?")) return input.issues as T; + if (route.endsWith("/agents")) return [{ id: "a1", runtimeConfig: { heartbeat: { maxConcurrentRuns: input.cap } } }] as T; + if (views.has(route)) { clock += input.viewMs; return views.get(route) as T; } + if (route.includes("assigned-work-invariant")) throw new Error("GET returned 404"); + return [] as T; + }, + post: async () => ({}) as T, + }; + const dir = await mkdtemp(path.join(os.tmpdir(), "hosted-cutoff-")); + dirs.push(dir); + const events: string[] = []; + const graded = await gradeHosted({ api, companyId: "c1", agentIds: ["a1"], releases: [], health: () => continuity([]), apiContinuity: () => continuity([]), + stopObserver: async () => {}, observedClicks: () => new Map(), lastReleaseAt: input.start - 1_000, settleMs: 1, drainMs: 1, dir, cancelled: () => false, + record: kind => events.push(kind), pollMs: 1, now: () => clock, quietMs: 90_000, tasks: () => input.tasks, wantedReleases: 0 }); + return { graded, events, json: JSON.parse(await readFile(path.join(dir, "hosted-release-swap.json"), "utf8")) }; + }; + + it("measures run activity against the moment the runs were read, not after minutes of task view reads", async () => { + const start = T0 + 10 * MIN; + const issues = [{ id: "i1", identifier: "GIT-740", title: "t", status: "in_progress", assigneeAgentId: "a1", createdAt: iso(T0) }, + ...Array.from({ length: 8 }, (_, n) => ({ id: `d${n}`, identifier: `GIT-${n}`, title: "d", status: "done", assigneeAgentId: "a1", createdAt: iso(T0), completedAt: iso(T0 + MIN) }))]; + const runs = (clock: number) => [apiRun("r1", "i1", T0, T0, { updatedAt: iso(clock - MIN) }), + ...issues.slice(1).map((issue, n) => apiRun(`f${n}`, issue.id, T0, T0, { status: "succeeded", finishedAt: iso(T0 + MIN), updatedAt: iso(T0 + MIN) }))]; + const { json } = await grade({ runs, issues, tasks: [], cap: 5, start, viewMs: MIN }); + const grade740 = json.grades.find((entry: Json) => entry.identifier === "GIT-740"); + expect(grade740).toMatchObject({ verdict: "still_working", label: null }); + }); + + it("does not grade subtasks still queued behind the agent's full slot cap, and fails the gate on the slot holders still running", async () => { + const start = T0 + 12 * MIN; + const issues = [ + { id: "p1", identifier: "GIT-743", title: "parent", status: "in_progress", assigneeAgentId: "a1", createdAt: iso(T0) }, + { id: "c1", identifier: "GIT-744", title: "SM sub 1", status: "todo", parentId: "p1", assigneeAgentId: "a1", createdAt: iso(T0 + 3 * MIN) }, + { id: "c2", identifier: "GIT-745", title: "SM sub 2", status: "todo", parentId: "p1", assigneeAgentId: "a1", createdAt: iso(T0 + 3 * MIN) }, + ]; + const runs = [apiRun("rp", "p1", T0, T0, { updatedAt: iso(start) }), apiRun("q1", "c1", T0 + 3 * MIN, null), apiRun("q2", "c2", T0 + 3 * MIN, null)]; + const { graded, events, json } = await grade({ runs, issues, tasks: [planned("p1", "GIT-743", T0)], cap: 1, start, viewMs: 0 }); + expect(json.notGraded).toEqual([{ issueId: "c1", identifier: "GIT-744", runIds: ["q1"] }, { issueId: "c2", identifier: "GIT-745", runIds: ["q2"] }]); + expect(json.grades.map((entry: Json) => entry.identifier)).toEqual(["GIT-743"]); + expect(json.r1.map((entry: Json) => entry.identifier)).toEqual(["GIT-743"]); + expect(events).toContain("hosted_not_graded_slot_queued"); + expect(graded.markdown).toContain("Not graded: 2 task(s) still queued behind a full agent slot cap"); + expect(graded.pass).toBe(false); + }); + + it("grades a queued subtask when the agent had a free slot, so a real stall still fails", async () => { + const start = T0 + 12 * MIN; + const issues = [{ id: "p1", identifier: "GIT-743", title: "parent", status: "in_progress", assigneeAgentId: "a1", createdAt: iso(T0) }, + { id: "c1", identifier: "GIT-744", title: "SM sub 1", status: "todo", parentId: "p1", assigneeAgentId: "a1", createdAt: iso(T0 + 3 * MIN) }]; + const runs = [apiRun("rp", "p1", T0, T0, { updatedAt: iso(start) }), apiRun("q1", "c1", T0 + 3 * MIN, null)]; + const { json } = await grade({ runs, issues, tasks: [planned("p1", "GIT-743", T0)], cap: 2, start, viewMs: 0 }); + expect(json.notGraded).toEqual([]); + expect(json.grades.find((entry: Json) => entry.identifier === "GIT-744")).toMatchObject({ verdict: "open_no_owner", label: "nothing pending" }); + }); +}); + +describe("the loop-guard notice", () => { + it("is informational when each copy came from a different run, and a duplicate otherwise", async () => { + expect(isNoticeRepeat(LOOP_GUARD_NOTICE, ["r1", "r2"])).toBe(true); + expect(isNoticeRepeat(LOOP_GUARD_NOTICE, ["r1", "r1"])).toBe(false); + expect(isNoticeRepeat(LOOP_GUARD_NOTICE, ["r1", null])).toBe(false); + expect(isNoticeRepeat(LOOP_GUARD_NOTICE, [])).toBe(false); + expect(isNoticeRepeat("DONE-c5164d03", ["r1", "r2"])).toBe(false); + expect(isNoticeRepeat("Prepared Ledger log for this response.", [null, null])).toBe(true); + const comments: Record = { + "/api/issues/i1/comments": [{ authorAgentId: "a1", createdByRunId: "r1", body: LOOP_GUARD_NOTICE }, { authorAgentId: "a1", createdByRunId: "r2", body: LOOP_GUARD_NOTICE }], + "/api/issues/i2/comments": [{ authorAgentId: "a1", createdByRunId: "r3", body: LOOP_GUARD_NOTICE }, { authorAgentId: "a1", createdByRunId: "r3", body: LOOP_GUARD_NOTICE }], + }; + const api = { get: async (route: string) => (comments[route] ?? []) as T }; + expect(await duplicateAgentComments(api, ["i1", "i2"])).toEqual({ duplicateComments: [{ issueId: "i2", count: 2 }], autoNoticeRepeats: [{ issueId: "i1", count: 2 }] }); + }); +}); diff --git a/tests/restart-chaos/hosted-gate.ts b/tests/restart-chaos/hosted-gate.ts index 3c880e5c384..ceafe7aa678 100644 --- a/tests/restart-chaos/hosted-gate.ts +++ b/tests/restart-chaos/hosted-gate.ts @@ -1,4 +1,5 @@ -import { agentSlotsFromRows, queueWaitFairnessFrom, type QueueWaitFairness } from "./agent-slots.js"; +import { agentSlotsFromRows, queueWaitFairnessFrom, slotFreeFrom, type QueueWaitFairness } from "./agent-slots.js"; +import type { SlotFree } from "./universal.js"; import { gradeR1 } from "./grader.js"; import type { ContinuityReport, DetectedRelease } from "./hosted-release.js"; import { ms, type HostedRunRow } from "./hosted-readers.js"; @@ -41,12 +42,27 @@ type R1Subject = Pick ({ id: agent.id, maxConcurrentRuns: agent.runtimeConfig?.heartbeat?.maxConcurrentRuns })), companyRuns.map(run => ({ id: run.runId, agentId: run.agentId, createdAt: run.createdAt, invocationSource: run.invocationSource, retryOfRunId: run.retryOfRunId, startedAt: run.startedAt, finishedAt: run.finishedAt }))); const capped = new Set(slots.agents.filter(agent => agent.maxConcurrentRuns !== null).map(agent => agent.agentId)); - return { queueWait: queueWaitFairnessFrom(slots), missing: agentIds.filter(id => !capped.has(id)) }; + return { queueWait: queueWaitFairnessFrom(slots), slotFree: slotFreeFrom(slots), missing: agentIds.filter(id => !capped.has(id)) }; +} + +export function slotQueuedRuns(runs: readonly HostedRunRow[], at: number, proof: { queueWait: QueueWaitFairness; slotFree: SlotFree }): Set { + return new Set(runs.filter(run => run.status === "queued" && run.startedAt === null && run.createdAt !== null && run.createdAt <= at + && proof.queueWait(run.runId, run.createdAt, at).verdict === "fair" && proof.slotFree(run.runId, at) === false).map(run => run.runId)); +} + +const SLOT_WAIT_STATUSES = new Set(["todo", "in_progress"]); +const LIVE_RUN_STATUSES = new Set(["queued", "running", "scheduled_retry"]); +const FAILED_RUN_STATUSES = new Set(["failed", "timed_out", "interrupted"]); + +export function queuedBehindSlotCap(status: unknown, runs: readonly HostedRunRow[], slotQueued: ReadonlySet): boolean { + if (!SLOT_WAIT_STATUSES.has(String(status)) || runs.some(run => FAILED_RUN_STATUSES.has(run.status))) return false; + const live = runs.filter(run => LIVE_RUN_STATUSES.has(run.status)); + return live.length > 0 && live.every(run => slotQueued.has(run.runId)); } /** diff --git a/tests/restart-chaos/hosted-grade.ts b/tests/restart-chaos/hosted-grade.ts index 696cd8e2c45..9ebeba5cf82 100644 --- a/tests/restart-chaos/hosted-grade.ts +++ b/tests/restart-chaos/hosted-grade.ts @@ -3,7 +3,7 @@ import { writeFile } from "node:fs/promises"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { requestBreakdown, type HostedApi, type RequestRecord } from "./hosted-api.js"; -import { hostedGate, hostedQueueWait, hostedR1, releaseOutages, subtaskRecord, type HostedAgentRow } from "./hosted-gate.js"; +import { hostedGate, hostedQueueWait, hostedR1, queuedBehindSlotCap, releaseOutages, slotQueuedRuns, subtaskRecord, type HostedAgentRow } from "./hosted-gate.js"; import { trackBanner, type HostedTaskRecord, type RetryBanner } from "./hosted-workload.js"; import { handsOffSummary, invariantLine, readAssignedWorkInvariant } from "./hosted-handsoff.js"; import { releaseMarkdown, type ContinuityReport, type DetectedRelease } from "./hosted-release.js"; @@ -55,9 +55,10 @@ export async function drainHosted(input: Pick { const now = input.now ?? Date.now; const settledAt = input.lastReleaseAt + input.settleMs; - const capAt = Math.max(settledAt, input.lastReleaseAt + input.drainMs); + const drainFrom = Math.max(input.lastReleaseAt, now()); + const capAt = Math.max(settledAt, drainFrom + input.drainMs); const window = PROGRESS_WINDOW_MS; - input.record("hosted_settling", { settleUntil: new Date(settledAt).toISOString(), capAt: new Date(capAt).toISOString() }); + input.record("hosted_settling", { settleUntil: new Date(settledAt).toISOString(), drainFrom: new Date(drainFrom).toISOString(), capAt: new Date(capAt).toISOString() }); while (!input.cancelled() && now() < settledAt) await delay(Math.min(input.pollMs ?? 15_000, Math.max(1, settledAt - now()))); for (;;) { const at = now(); @@ -118,6 +119,7 @@ export async function gradeHosted(input: HostedGradeInput): Promise<{ headline: issues = startedIssues(all, runs); await readViews(issues.map(issue => issue.id)); } + const gradedAt = now(); runs = await hostedCompanyRuns(api, companyId, agentIds); const allIssues = await hostedIssues(api, companyId, agentIds); issues = startedIssues(allIssues, runs); @@ -125,10 +127,20 @@ export async function gradeHosted(input: HostedGradeInput): Promise<{ headline: const person = await pendingPersonItems(api, issues.map(issue => issue.id)); const runsByIssue = new Map(); for (const run of runs) if (run.issueId) runsByIssue.set(run.issueId, [...(runsByIssue.get(run.issueId) ?? []), run]); - const gradedAt = now(); + const agents = (await api.get(`/api/companies/${companyId}/agents`).catch(error => { + input.record("hosted_agents_unavailable", { error: String(error) }); + return [] as HostedAgentRow[]; + })).filter(agent => agentIds.includes(String(agent.id))); + const slotProof = hostedQueueWait(agentIds, agents, runs); + if (slotProof.missing.length) input.record("hosted_agent_slots_unavailable", { agentIds: slotProof.missing }); + const slotQueued = slotQueuedRuns(runs, gradedAt, slotProof); + const notGraded = issues.filter(issue => queuedBehindSlotCap(views.get(issue.id)?.status, runsByIssue.get(issue.id) ?? [], slotQueued)) + .map(issue => ({ issueId: issue.id, identifier: issue.identifier, runIds: (runsByIssue.get(issue.id) ?? []).filter(run => slotQueued.has(run.runId)).map(run => run.runId) })); + const notGradedIds = new Set(notGraded.map(entry => entry.issueId)); + if (notGraded.length) input.record("hosted_not_graded_slot_queued", { tasks: notGraded }); const unreadable = issues.filter(issue => (views.get(issue.id) ?? { status: "unreadable" }).status === "unreadable" || person.unreadable.includes(issue.id)); if (unreadable.length) input.record("hosted_views_unreadable", { count: unreadable.length, issueIds: unreadable.map(issue => issue.id) }); - const grades: TaskGrade[] = issues.filter(issue => !unreadable.includes(issue)).map(issue => gradeReleaseSwapTask({ + const grades: TaskGrade[] = issues.filter(issue => !unreadable.includes(issue) && !notGradedIds.has(issue.id)).map(issue => gradeReleaseSwapTask({ now: gradedAt, staleAfterMs: PROGRESS_WINDOW_MS, issueId: issue.id, identifier: issue.identifier, title: issue.title, ...issueViewFields(views.get(issue.id)!), runs: gradeRuns(runsByIssue.get(issue.id) ?? []), queuedWakes: 0, retryClicks: clicks.get(issue.id) ?? [], pendingApproval: person.approvals.has(issue.id), pendingQuestion: person.questions.has(issue.id) })); @@ -158,17 +170,11 @@ export async function gradeHosted(input: HostedGradeInput): Promise<{ headline: const health = input.health(); const apiContinuity = input.apiContinuity(); const handsOffPass = result.uncollected === 0 && result.stillRunning === 0 && result.started > 0 && result.completedWithoutHuman === result.started; - const agents = (await api.get(`/api/companies/${companyId}/agents`).catch(error => { - input.record("hosted_agents_unavailable", { error: String(error) }); - return [] as HostedAgentRow[]; - })).filter(agent => agentIds.includes(String(agent.id))); - const slotProof = hostedQueueWait(agentIds, agents, runs); - if (slotProof.missing.length) input.record("hosted_agent_slots_unavailable", { agentIds: slotProof.missing }); const outages = releaseOutages(input.releases); - const started = new Set(issues.map(issue => issue.id)); + const started = new Set(issues.filter(issue => !notGradedIds.has(issue.id)).map(issue => issue.id)); const planned = input.tasks(); const plannedIds = new Set(planned.flatMap(task => task.issueId ? [task.issueId] : [])); - const subtasks = issues.filter(issue => !plannedIds.has(issue.id) && issue.parentId !== null && plannedIds.has(issue.parentId)).map(issue => subtaskRecord({ ...views.get(issue.id), id: issue.id, + const subtasks = issues.filter(issue => started.has(issue.id) && !plannedIds.has(issue.id) && issue.parentId !== null && plannedIds.has(issue.parentId)).map(issue => subtaskRecord({ ...views.get(issue.id), id: issue.id, identifier: issue.identifier }, clicks.get(issue.id) ?? [])); const r1 = [...planned.filter(task => task.issueId && started.has(task.issueId)), ...subtasks].map(task => ({ identifier: task.identifier, result: hostedR1({ record: task, view: views.get(task.issueId!), runs: runsByIssue.get(task.issueId!) ?? [], queueWait: slotProof.queueWait, outages }) })); @@ -184,6 +190,8 @@ export async function gradeHosted(input: HostedGradeInput): Promise<{ headline: invariantLine(invariant), "", ...gate.lines, + ...(notGraded.length ? [`Not graded: ${notGraded.length} task(s) still queued behind a full agent slot cap when the grade read the runs, proven by the agent's slot evidence ` + + `(${notGraded.map(entry => entry.identifier ?? entry.issueId).join(", ")}).`] : []), "", releaseSwapMarkdown(result, metrics, windows.map(() => null), duplicates, grades, failedOutsideSwaps(windows, facts)).trimEnd(), "", @@ -195,7 +203,7 @@ export async function gradeHosted(input: HostedGradeInput): Promise<{ headline: ...UNAVAILABLE_OVER_API.map(item => `- ${item}`), "", ].join("\n"); - await writeFile(path.join(input.dir, "hosted-release-swap.json"), `${JSON.stringify({ schema: "paperclip.restart-chaos.hosted-release-swap/v1", result, handsOff: { ...handsOff, banners: Object.fromEntries(banners) }, gate, r1, invariant, + await writeFile(path.join(input.dir, "hosted-release-swap.json"), `${JSON.stringify({ schema: "paperclip.restart-chaos.hosted-release-swap/v1", result, handsOff: { ...handsOff, banners: Object.fromEntries(banners) }, gate, r1, notGraded, invariant, releases: input.releases, health, api: apiContinuity, requestBreakdown: breakdown, metrics, duplicates, failedRuns, grades, drained, unavailable: UNAVAILABLE_OVER_API }, null, 2)}\n`, { mode: 0o600 }); await writeFile(path.join(input.dir, "SUMMARY.md"), markdown, { mode: 0o600 }); diff --git a/tests/restart-chaos/hosted-readers.ts b/tests/restart-chaos/hosted-readers.ts index 6d7cb804618..69fe44e5c25 100644 --- a/tests/restart-chaos/hosted-readers.ts +++ b/tests/restart-chaos/hosted-readers.ts @@ -1,6 +1,6 @@ import { wakeInputCommentIds } from "./follow-up-delivery.js"; import type { HostedApi } from "./hosted-api.js"; -import { isAutoNotice, type GradeRun, type GradeTaskInput } from "./release-swap-grade.js"; +import { isNoticeRepeat, type GradeRun, type GradeTaskInput } from "./release-swap-grade.js"; type Json = Record; export type ReadApi = Pick; @@ -168,12 +168,16 @@ export async function duplicateAgentComments(api: ReadApi, issueIds: readonly st const autoNoticeRepeats: Array<{ issueId: string; count: number }> = []; for (const id of issueIds) { const comments = await api.get(`/api/issues/${id}/comments`).catch(() => [] as Json[]); - const counts = new Map(); + const counts = new Map }>(); for (const comment of comments) if (comment.authorAgentId) { const key = `${comment.authorAgentId}\u0000${comment.body}`; - counts.set(key, { body: String(comment.body ?? ""), count: (counts.get(key)?.count ?? 0) + 1 }); + const entry = counts.get(key) ?? { body: String(comment.body ?? ""), runIds: [] }; + entry.runIds.push(text(comment.createdByRunId)); + counts.set(key, entry); + } + for (const { body, runIds } of counts.values()) if (runIds.length > 1) { + (isNoticeRepeat(body, runIds) ? autoNoticeRepeats : duplicateComments).push({ issueId: id, count: runIds.length }); } - for (const { body, count } of counts.values()) if (count > 1) (isAutoNotice(body) ? autoNoticeRepeats : duplicateComments).push({ issueId: id, count }); } return { duplicateComments, autoNoticeRepeats }; } diff --git a/tests/restart-chaos/release-swap-finish.test.ts b/tests/restart-chaos/release-swap-finish.test.ts index d5befa66422..3086aea8527 100644 --- a/tests/restart-chaos/release-swap-finish.test.ts +++ b/tests/restart-chaos/release-swap-finish.test.ts @@ -87,6 +87,17 @@ describe("release swap end-of-wave grade", () => { expect(capped.result.pass).toBe(false); }); + it("counts a loop-guard notice from different runs as informational and the same notice twice from one run as a duplicate", async () => { + const notice = "Paperclip stopped a tool call that kept failing the same way. The task continues in a fresh session."; + const base = fakeDb({ issues: [{ id: "d1", identifier: "CHA-5", title: "d" }], runs: () => [{ runId: "r1", issueId: "d1", status: "succeeded" }] }); + const db = { $client: { unsafe: async (sql: string, params?: unknown[]) => sql.includes("from issue_comments") + ? [{ issueId: "d1", body: notice, count: 2, runIds: ["r1", "r2"] }, { issueId: "d2", body: notice, count: 2, runIds: ["r3", "r3"] }] + : base.$client.unsafe(sql, params as never) } } as unknown as ReleaseSwapFinishInput["db"]; + const { json } = await finish({ db, api: { get: async () => ({ status: "done" }), post: async () => ({}) } as unknown as ReleaseSwapFinishInput["api"] }); + expect(json.duplicates.autoNoticeRepeats).toEqual([{ issueId: "d1", count: 2 }]); + expect(json.duplicates.duplicateComments).toEqual([{ issueId: "d2", count: 2 }]); + }); + it("grades a pending question from the database as a person's wait, and passes a clean wave", async () => { const { json } = await finish({ api: { get: async () => ({ status: "blocked" }), post: async () => ({}) } as unknown as ReleaseSwapFinishInput["api"], db: fakeDb({ issues: [{ id: "q1", identifier: "CHA-3", title: "q" }], runs: () => [{ runId: "r1", issueId: "q1", status: "succeeded" }], questions: ["q1"] }) }); diff --git a/tests/restart-chaos/release-swap-finish.ts b/tests/restart-chaos/release-swap-finish.ts index 5c346ede12e..de6b2d85d39 100644 --- a/tests/restart-chaos/release-swap-finish.ts +++ b/tests/restart-chaos/release-swap-finish.ts @@ -6,7 +6,7 @@ import type { ChaosApi } from "./api.js"; import { checkCrossServer, readCrossServerEvidence, type ServerProcessWindow } from "./multi-server.js"; import { RUN_OWNER_BOOT_ID, type ReleaseSwapDriver } from "./release-swap-driver.js"; import { - drainState, failedOutsideSwaps, gradeReleaseSwapTask, isAutoNotice, overlappingRuns, QUIET_PERIOD_MS, releaseSwapMarkdown, summarizeGrades, swapForFailedRun, swapMetrics, + drainState, failedOutsideSwaps, gradeReleaseSwapTask, isNoticeRepeat, overlappingRuns, QUIET_PERIOD_MS, releaseSwapMarkdown, summarizeGrades, swapForFailedRun, swapMetrics, type DuplicateFacts, type GradeTaskInput, type RetryClick, type SwapRunFact, type TaskGrade, } from "./release-swap-grade.js"; import { runsStartedDuringSwap, swapMoments, swapMomentsMarkdown } from "./release-swap-moments.js"; @@ -175,10 +175,11 @@ async function duplicateFacts(input: ReleaseSwapFinishInput, facts: readonly Swa const cross = await readCrossServerEvidence(input.db as Db, input.companyId) .then(evidence => checkCrossServer({ ...evidence, windows: input.serverWindows })).catch(() => null); const repeated = await input.db.$client.unsafe( - `select issue_id::text as "issueId", body, count(*)::int as count from issue_comments + `select issue_id::text as "issueId", body, count(*)::int as count, array_agg(created_by_run_id::text) as "runIds" from issue_comments where company_id::text = $1 and author_agent_id is not null group by issue_id, author_agent_id, body having count(*) > 1`, [input.companyId]) as Row[]; - const comments = repeated.filter(row => !isAutoNotice(String(row.body ?? ""))); - const notices = repeated.filter(row => isAutoNotice(String(row.body ?? ""))); + const notice = (row: Row) => isNoticeRepeat(String(row.body ?? ""), Array.isArray(row.runIds) ? row.runIds as Array : []); + const comments = repeated.filter(row => !notice(row)); + const notices = repeated.filter(notice); const children = await input.db.$client.unsafe( `select parent_id::text as "parentId", title, count(*)::int as count from issues where company_id::text = $1 and parent_id is not null and hidden_at is null group by parent_id, title having count(*) > 1`, [input.companyId]) as Row[]; diff --git a/tests/restart-chaos/release-swap-grade.ts b/tests/restart-chaos/release-swap-grade.ts index b8c0568ab1c..9b9b27a3ff2 100644 --- a/tests/restart-chaos/release-swap-grade.ts +++ b/tests/restart-chaos/release-swap-grade.ts @@ -239,6 +239,14 @@ export function drainState(runs: readonly DrainRun[], now: number, windowMs: num export const AUTO_NOTICE = /^Prepared .+ for this response\.$/; export const isAutoNotice = (body: string) => AUTO_NOTICE.test(body.trim()); +export const LOOP_GUARD_NOTICE = "Paperclip stopped a tool call that kept failing the same way. The task continues in a fresh session."; + +export function isNoticeRepeat(body: string, runIds: ReadonlyArray): boolean { + if (isAutoNotice(body)) return true; + if (body.trim() !== LOOP_GUARD_NOTICE) return false; + const known = runIds.filter((id): id is string => typeof id === "string" && id.length > 0); + return runIds.length > 1 && known.length === runIds.length && new Set(known).size === known.length; +} export function swapMetrics(swaps: readonly SwapWindow[], runs: readonly SwapRunFact[], firstEventAfterStop: ReadonlyMap, waveEnd: number, failureWindowMs = RELEASE_FAILURE_WINDOW_MS): SwapMetrics[] { @@ -347,7 +355,7 @@ export function releaseSwapMarkdown(result: ReleaseSwapResult, metrics: readonly `Retry: ${result.retry.tasks} task(s) needed it; worked ${result.retry.worked}, did not work ${result.retry.didNotWork}, still running ${result.retry.pending}.`, `Duplicates: ${duplicates.overlappingRuns.length} overlapping run pair(s), ${duplicates.doubleExecutors.length} double executor(s), ` + `${duplicates.duplicateComments.length} duplicated agent comment(s), ${duplicates.duplicateChildren.length} duplicated subtask title(s)` - + `${duplicates.autoNoticeRepeats ? `; informational: ${duplicates.autoNoticeRepeats.length} task(s) repeat the "Prepared ... for this response." notice` : ""}.`, + + `${duplicates.autoNoticeRepeats ? `; informational: ${duplicates.autoNoticeRepeats.length} task(s) repeat the "Prepared ... for this response." notice or a once-per-run "stopped a tool call" notice` : ""}.`, `Failed outside a release window (more than ${RELEASE_FAILURE_WINDOW_MS / 60_000} min after a stop and not live at it): ${failedOutside.length ? failedOutside.join(", ") : "none"}.`, "", "| Swap | Failed runs | Lost while old alive | Survivors | Next event p50 | Next event max | No event | Late start on old | Prep during recovery |",