From 558dbb01320987eb9aba80d5d1a8e66604456f72 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sat, 21 Nov 2026 07:33:25 -0600 Subject: [PATCH] test(restart-chaos): grade hosted runs with R1 and a strict release-swap gate (#22490) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The restart-chaos harness (`tests/restart-chaos`) proves that agent work survives a control-plane release swap > - Local waves grade each task with rules R1–R6. R1 is the time budget, with credit for a provably fair queue wait (#22446, #22452) > - Hosted mode (`hosted.ts`) runs the same proof on a real test staging instance that takes automatic releases > - Hosted mode passed a run when every task finished hands-off. It did not apply R1, and banners, failed health probes, duplicates and the swap shape were only reported > - The next acceptance gate needs 10 consecutive hosted releases with 100% hands-off, 0 Retry banners and 0 failed health probes, so the exit code must enforce these rules > - This pull request adds hosted R1 and a strict "Gate D" verdict to the hosted grade > - The benefit is that one hosted run gives a single pass or fail with the same R1 rule as the local waves ## Linked Issues or Issue Description Refs #22446 Refs #22452 Refs #21153 **What happened?** A hosted run (`hosted.ts --hosted-releases N`) exited 0 when every started task finished without a person. The run did not grade R1. A Retry banner that cleared by itself, a failed `/api/health` probe, a duplicated agent comment, a stop-then-start release or a release with no live run did not fail the run. These were only lines in `SUMMARY.md`. **Expected behavior** A hosted run passes only when every task finishes hands-off and inside the R1 budget (with the same fair queue-wait credit as the local waves), no Retry banner shows, no health probe fails, no work is duplicated, no release is stop-then-start, and each release carries at least one live run. **Steps to reproduce** 1. Run `hosted.ts --hosted-releases 1` against a test staging instance whose deploy is stop-then-start. 2. Let all tasks finish by themselves after the outage. 3. See exit 0, although health probes failed during the release. **Paperclip version or commit** `95ba0a7de6` (master) ## What Changed - `hosted-gate.ts` (new): `hostedR1` builds the R1 input from API rows and calls `gradeR1` from `grader.ts`. It uses the staging-mix budget (`stagingMixBudgetMs`), outage time only for a release whose health probes failed before the new build answered, and the fair queue-wait proof (`queueWaitFairnessFrom`) from the company run list and each agent's `maxConcurrentRuns`. A follow-up run's wait is not credited, because the API has no wake evidence. `hostedGate` lists seven checks and passes only when all hold. - `hosted-readers.ts`: run rows keep `invocationSource`, `retryOfRunId` and the wake comment ids. - `hosted-grade.ts`: grades every started task with R1, adds the `Gate D` block to `SUMMARY.md`, adds `gate` and `r1` to `hosted-release-swap.json`, and returns the gate as the pass. - `hosted-main.ts`: passes the task records to the grade. - `README.md`: documents the gate and the API proxies for per-release hand-off and reach. ## Verification - `npx vitest run --config tests/restart-chaos/vitest.config.ts tests/restart-chaos/hosted` passes (8 files, 56 tests). The new `hosted-gate.test.ts` covers R1 pass and fail, a fair and an unfair queue wait, outage credit, Retry and open tasks, and each gate check. - `npx tsc -p tests/restart-chaos/tsconfig.json --noEmit` is clean. - No hosted run was started for this change. It changes only how a finished run is graded. ## Risks - Low risk. The change is in the test harness only. - Hosted runs that passed before can now fail on R1, a banner, a failed probe, a duplicate, a stop-then-start release or an unloaded release. This is intended. - A follow-up run's queue wait gets no credit on hosted runs, so R1 can be stricter than in local waves when follow-ups queue for a long time. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, used in Claude Code with tool use and code execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- tests/restart-chaos/README.md | 13 ++ tests/restart-chaos/hosted-drain.test.ts | 2 +- tests/restart-chaos/hosted-gate.test.ts | 215 +++++++++++++++++++++ tests/restart-chaos/hosted-gate.ts | 123 ++++++++++++ tests/restart-chaos/hosted-grade.ts | 31 ++- tests/restart-chaos/hosted-main.test.ts | 2 +- tests/restart-chaos/hosted-main.ts | 1 + tests/restart-chaos/hosted-readers.test.ts | 2 +- tests/restart-chaos/hosted-readers.ts | 7 +- 9 files changed, 388 insertions(+), 8 deletions(-) create mode 100644 tests/restart-chaos/hosted-gate.test.ts create mode 100644 tests/restart-chaos/hosted-gate.ts diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 636e56f5a27..50878cbcca7 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -305,6 +305,19 @@ run list when that list is full, so no task's early runs are lost. A queued or r minutes is not a live owner, the same window the drain uses. The drain also waits for the 90 s quiet period (`--hosted-quiet-seconds`), and the grade reads runs and task views again immediately before grading. +**Gate D verdict** (`hosted-gate.ts`). A run exits 0 only when every check in the `Gate D` block of `SUMMARY.md` holds: +every started task completed hands-off; every task the harness launched was created; every started task passes the local waves' +R1 (`gradeR1` from `grader.ts`); no Retry banner was shown; no health probe failed; no overlapping runs, duplicated +agent comments or duplicated subtasks; no release was stop-then-start; and at least one release was detected, each +with a live run carried across it. A fixed batch (`--hosted-releases 0`) skips the release check. R1 uses the +staging-mix budget (`stagingMixBudgetMs`); a subtask the agent created under a launched task is graded from its own creation time with the +budget of its two rounds. It adds outage time only for a release whose health probes failed +before the new build answered. It credits the first run's queue wait when the agent's slots prove it fair +(`queueWaitFairnessFrom`, from the run list and each agent's `runtimeConfig.heartbeat.maxConcurrentRuns`). The API +has no wake evidence, so a follow-up run's queue wait is never credited. Per-release hand-off and reach are API +proxies: runs alive at the old build's last answer, and how many of them wrote an event after it. The server log's +`handed live native runs` and `reattach phase ended` lines are not readable through the tenant API. + These fields are unavailable over the API and are reported as unavailable, not guessed: | Field | Why | diff --git a/tests/restart-chaos/hosted-drain.test.ts b/tests/restart-chaos/hosted-drain.test.ts index a9e670e4355..0a30e7342f6 100644 --- a/tests/restart-chaos/hosted-drain.test.ts +++ b/tests/restart-chaos/hosted-drain.test.ts @@ -47,7 +47,7 @@ describe("end-of-wave drain", () => { }; const graded = await gradeHosted({ api, companyId: "c1", agentIds: ["a1"], releases: [], health: () => continuity([]), apiContinuity: () => continuity([]), stopObserver: async () => {}, observedClicks: () => new Map(), lastReleaseAt: ended - 1_000, settleMs: 1, drainMs: 3_600_000, dir, - cancelled: () => false, record: () => {}, pollMs: 1, now: () => clock, quietMs: 90_000 }); + cancelled: () => false, record: () => {}, pollMs: 1, now: () => clock, quietMs: 90_000, tasks: () => [], wantedReleases: 0 }); const json = JSON.parse(await readFile(path.join(dir, "hosted-release-swap.json"), "utf8")); expect(json.grades.map((grade: Json) => [grade.identifier, grade.verdict])).toEqual([["GIT-87", "pass"]]); expect(graded.headline).toBe("1 / 1 tasks completed hands-off"); diff --git a/tests/restart-chaos/hosted-gate.test.ts b/tests/restart-chaos/hosted-gate.test.ts new file mode 100644 index 00000000000..85bb6a8954d --- /dev/null +++ b/tests/restart-chaos/hosted-gate.test.ts @@ -0,0 +1,215 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { hostedGate, hostedQueueWait, hostedR1, releaseOutages, subtaskRecord } from "./hosted-gate.js"; +import { gradeHosted } from "./hosted-grade.js"; +import { continuity } from "./hosted-release.js"; +import { runRow, type HostedRunRow } from "./hosted-readers.js"; +import type { DetectedRelease } from "./hosted-release.js"; +import type { HostedTaskRecord } from "./hosted-workload.js"; +import type { SwapMetrics } from "./release-swap-grade.js"; +import { roundsFor, stagingMixBudgetMs, SUBTASK_ROUNDS } from "./staging-mix.js"; + +const dirs: string[] = []; +afterEach(async () => { for (const dir of dirs.splice(0)) await rm(dir, { recursive: true, force: true }); }); + +const T0 = Date.parse("2026-11-21T10:00:00.000Z"); +const MIN = 60_000; +const at = (minutes: number) => new Date(T0 + minutes * MIN).toISOString(); +const task = { turnMinutes: 5, rounds: roundsFor(5), subtasks: 0, followUpAfterMs: null, arriveAtMs: 0, holdMs: 0 }; +const BUDGET_MIN = stagingMixBudgetMs(task) / MIN; + +function record(overrides: Partial = {}): HostedTaskRecord { + return { plan: { index: 0, adapter: "opencode", nonce: "abcd1234", task }, issueId: "i1", identifier: "GIT-1", createdAt: T0, createError: null, + status: "done", lastLiveAt: null, followUp: null, retryClicks: [], retriedRuns: [], doneAt: null, ...overrides }; +} + +const run = (id: string, created: number, started: number | null, finished: number | null, extra: Record = {}): HostedRunRow => + runRow({ id, agentId: "a1", issueId: "i1", status: finished === null ? "running" : "succeeded", createdAt: at(created), startedAt: started === null ? null : at(started), + finishedAt: finished === null ? null : at(finished), invocationSource: "assignment", ...extra }); + +const agents = [{ id: "a1", runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } } }]; +const qw = (companyRuns: HostedRunRow[]) => hostedQueueWait(["a1"], agents, companyRuns).queueWait; + +function release(overrides: Partial = {}): DetectedRelease { + return { index: 1, from: { processStartedAt: "a", commit: "c1" }, to: { processStartedAt: "b", commit: "c2" }, commitChanged: true, lastOldOnlyAt: T0, + firstNewAt: T0 + MIN, lastOldAt: T0 + 2 * MIN, overlapWindow: { from: T0 + MIN, to: T0 + 2 * MIN }, shape: "overlap", failedRequests: 0, slowRequests: 0, + firstFailureAt: null, timeToFirstNewMs: MIN, settled: true, ...overrides }; +} + +describe("hosted R1", () => { + it("passes a task done inside the staging-mix budget and fails one done after it with no fair queue wait", () => { + const fast = hostedR1({ record: record(), view: { status: "done", completedAt: at(10) }, runs: [run("r1", 0, 0, 10)], queueWait: qw([run("r1", 0, 0, 10)]), outages: [] }); + expect(fast.verdict).toBe("pass"); + const late = BUDGET_MIN + 5; + const slow = hostedR1({ record: record(), view: { status: "done", completedAt: at(late) }, runs: [run("r1", 0, 0, late)], queueWait: qw([run("r1", 0, 0, late)]), outages: [] }); + expect(slow).toMatchObject({ verdict: "fail" }); + expect(slow.detail).toMatch(/^done_too_late/); + }); + + it("credits the first run's queue wait when the agent's only slot was busy the whole time", () => { + const wait = 30; + const done = wait + BUDGET_MIN - 5; + const busy = run("other", -1, -1, wait, { issueId: "i0" }); + const mine = run("r1", 0, wait, done); + const graded = hostedR1({ record: record(), view: { status: "done", completedAt: at(done) }, runs: [mine], queueWait: qw([busy, mine]), outages: [] }); + expect(graded.verdict).toBe("pass"); + expect(graded.detail).toContain("fair queue wait"); + }); + + it("fails an unfair wait: a newer assignment run of the same agent started ahead of the task's run", () => { + const wait = 30; + const done = wait + BUDGET_MIN - 5; + const overtaker = run("newer", 1, 2, wait, { issueId: "i0" }); + const mine = run("r1", 0, wait, done); + const graded = hostedR1({ record: record(), view: { status: "done", completedAt: at(done) }, runs: [mine], queueWait: qw([overtaker, mine]), outages: [] }); + expect(graded.verdict).toBe("fail"); + expect(graded.detail).toMatch(/^unfair_queue_wait/); + }); + + it("credits outage time only for a release whose health probes failed before the new build answered", () => { + expect(releaseOutages([release()])).toEqual([]); + const outage = releaseOutages([release({ shape: "stop_then_start", firstFailureAt: T0 + 10 * MIN, firstNewAt: T0 + 20 * MIN })]); + expect(outage).toHaveLength(1); + expect(outage[0]).toMatchObject({ startedAt: T0 + 10 * MIN, readyAt: T0 + 20 * MIN }); + const done = BUDGET_MIN + 5; + const graded = hostedR1({ record: record(), view: { status: "done", completedAt: at(done) }, runs: [run("r1", 0, 0, done)], queueWait: qw([run("r1", 0, 0, done)]), outages: outage }); + expect(graded.verdict).toBe("pass"); + }); + + it("never credits a follow-up run's queue wait, because the API has no wake evidence", () => { + const first = run("r1", 0, 0, 5); + const busy = run("other", 4, 4, 40, { issueId: "i0" }); + const follow = run("r2", 6, 40, 45, { invocationSource: "automation", contextSnapshot: { wakeCommentIds: ["c1"] } }); + const done = 45; + expect(done).toBeGreaterThan(BUDGET_MIN); + const graded = hostedR1({ record: record({ followUp: { at: T0 + 6 * MIN - 1, commentId: "c1", error: null } }), view: { status: "done", completedAt: at(done) }, + runs: [first, follow], queueWait: qw([first, busy, follow]), outages: [] }); + expect(graded.verdict).toBe("fail"); + expect(graded.detail).toContain("no wake evidence for follow-up run r2"); + }); + + it("names an agent whose maxConcurrentRuns cannot be read", () => { + expect(hostedQueueWait(["a1", "a2"], [agents[0]!, { id: "a2", runtimeConfig: null }], []).missing).toEqual(["a2"]); + expect(hostedQueueWait(["a1"], [], []).missing).toEqual(["a1"]); + }); + + it("marks a task that needed Retry n/a for R1 and fails one that did not rest done", () => { + const retried = hostedR1({ record: record({ retryClicks: [{ at: T0, accepted: true, via: "grader" }] }), view: { status: "done", completedAt: at(10) }, + runs: [], queueWait: qw([]), outages: [] }); + expect(retried.verdict).toBe("n/a"); + const open = hostedR1({ record: record(), view: { status: "in_progress" }, runs: [run("r1", 0, 0, null)], queueWait: qw([]), outages: [] }); + expect(open.detail).toMatch(/^rested_not_done/); + }); +}); + +describe("gate D verdict", () => { + const metric = (index: number, survivors: number): SwapMetrics => ({ index, failedRuns: [], lostWhileOldAlive: [], + survivors: Array.from({ length: survivors }, (_, n) => ({ runId: `r${n}`, nextEventAfterMs: 1_000 })), nextEventP50Ms: 1_000, nextEventMaxMs: 1_000, survivorsWithoutEvent: 0 }); + const clean = { handsOffPass: true, wantedReleases: 1, createErrors: 0, slotEvidenceMissing: [] as string[], r1: [{ identifier: "GIT-1", result: { verdict: "pass" as const, detail: "done" } }], bannersShown: 0, + health: { requests: 100, failures: 0, slow: 0, longestGapMs: 2_000, longestGap: null }, + duplicates: { overlappingRuns: [], duplicateComments: [], duplicateChildren: [] }, releases: [release()], metrics: [metric(1, 3)] }; + + it("passes only when every criterion holds", () => { + const gate = hostedGate(clean); + expect(gate.pass).toBe(true); + expect(gate.lines[0]).toBe("Gate D: PASS"); + }); + + it.each([ + ["a banner", { bannersShown: 1 }, "Retry banners"], + ["a failed health probe", { health: { ...clean.health, failures: 1 } }, "health probes"], + ["a duplicated comment", { duplicates: { ...clean.duplicates, duplicateComments: [{ issueId: "i1", count: 2 }] } }, "duplicates"], + ["a stop-then-start release", { releases: [release({ shape: "stop_then_start" })] }, "swap shape"], + ["a release with no live run", { metrics: [metric(1, 0)] }, "releases under load"], + ["no release", { releases: [], metrics: [] }, "releases under load"], + ["an R1 failure", { r1: [{ identifier: "GIT-2", result: { verdict: "fail" as const, detail: "done_too_late 9s > 1s" } }] }, "R1"], + ["a task that was not hands-off", { handsOffPass: false }, "hands-off"], + ["a task that could not be created", { createErrors: 1 }, "tasks created"], + ["unreadable agent slots", { slotEvidenceMissing: ["a1"] }, "agent slot evidence"], + ])("fails on %s", (_name, change, check) => { + const gate = hostedGate({ ...clean, ...change }); + expect(gate.pass).toBe(false); + expect(gate.lines.find(line => line.includes(`FAIL ${check}`))).toBeDefined(); + }); + + it("does not require a release for a fixed batch (--hosted-releases 0)", () => { + const gate = hostedGate({ ...clean, wantedReleases: 0, releases: [], metrics: [] }); + expect(gate.pass).toBe(true); + expect(gate.lines.some(line => line.includes("releases under load"))).toBe(false); + }); +}); + +describe("hosted grade with the run's task records", () => { + const issue = (id: string, extra: Record = {}) => ({ id, identifier: id.toUpperCase(), title: id, status: "done", assigneeAgentId: "a1", + createdAt: at(0), completedAt: at(10), ...extra }); + const apiFor = (issues: Json[], runs: Json[]) => ({ + get: async (route: string) => (route.includes("heartbeat-runs") ? runs : route.includes("/issues?") ? issues + : route.startsWith("/api/issues/") && !route.slice("/api/issues/".length).includes("/") ? issues.find(entry => route.endsWith(`/${entry.id}`)) ?? { status: "unreadable" } + : route.endsWith("/agents") ? (agentList ?? (() => { throw new Error("forbidden"); })()) : []) as T, + post: async () => ({}) as T, + }); + type Json = Record; + let events: string[] = []; + let agentList: Json[] | null = agents; + const grade = async (issues: Json[], runs: Json[], tasks: HostedTaskRecord[], wantedReleases: number) => { + events = []; + const dir = await mkdtemp(path.join(os.tmpdir(), "hosted-gate-")); + dirs.push(dir); + return gradeHosted({ api: apiFor(issues, runs), companyId: "c1", agentIds: ["a1"], releases: [], health: () => continuity([]), apiContinuity: () => continuity([]), + stopObserver: async () => {}, observedClicks: () => new Map(), lastReleaseAt: Date.now() - 1_000, settleMs: 5, drainMs: 5, dir, cancelled: () => false, + record: event => events.push(event), pollMs: 5, tasks: () => tasks, wantedReleases }); + }; + const runOf = (id: string, issueId: string, created: number, finished: number) => ({ id, agentId: "a1", issueId, status: "succeeded", invocationSource: "assignment", + createdAt: at(created), startedAt: at(created), finishedAt: at(finished) }); + + it("passes a healthy fixed batch with no release", async () => { + const graded = await grade([issue("i1")], [runOf("r1", "i1", 0, 10)], [record()], 0); + expect(graded.pass).toBe(true); + expect(graded.markdown).toContain("Gate D: PASS"); + }); + + it("grades an agent-created subtask on its own two-round budget", async () => { + const budget = stagingMixBudgetMs({ ...task, rounds: SUBTASK_ROUNDS }) / MIN; + const late = budget + 5; + const child = issue("c1", { parentId: "i1", createdAt: at(1), completedAt: at(1 + late) }); + const graded = await grade([issue("i1", { completedAt: at(late + 2) }), child], [runOf("r1", "i1", 0, 3), runOf("r2", "c1", 1, 1 + late), runOf("r3", "i1", late + 1, late + 2)], + [record({ plan: { ...record().plan, task: { ...task, subtasks: 2 } } })], 0); + expect(graded.pass).toBe(false); + expect(graded.markdown).toMatch(/FAIL R1: 1 failed \(C1: done_too_late/); + }); + + it("builds a subtask record from its issue view", () => { + const child = subtaskRecord({ id: "c1", identifier: "GIT-9", status: "done", createdAt: at(3) }, []); + expect(child).toMatchObject({ issueId: "c1", identifier: "GIT-9", createdAt: T0 + 3 * MIN, createError: null }); + expect(child.plan.task.rounds).toBe(SUBTASK_ROUNDS); + expect(subtaskRecord({ id: "c2", status: "done" }, []).createError).toBe("subtask has no createdAt"); + expect(subtaskRecord({ id: "c3", status: "done", createdAt: T0 }, []).createdAt).toBe(T0); + }); + + it("reads a numeric completedAt as well as an ISO string", () => { + const graded = hostedR1({ record: record(), view: { status: "done", completedAt: T0 + 10 * MIN }, runs: [run("r1", 0, 0, 10)], queueWait: qw([run("r1", 0, 0, 10)]), outages: [] }); + expect(graded.verdict).toBe("pass"); + }); + + it("grades only subtasks of launched tasks, not another issue the agent created", async () => { + const budget = stagingMixBudgetMs({ ...task, rounds: SUBTASK_ROUNDS }) / MIN; + const late = budget + 5; + const stray = issue("x1", { createdAt: at(1), completedAt: at(1 + late) }); + const graded = await grade([issue("i1"), stray], [runOf("r1", "i1", 0, 10), runOf("r2", "x1", 1, 1 + late)], [record()], 0); + expect(graded.markdown).not.toContain("FAIL R1"); + }); + + it("fails the gate and records why when the agent list cannot be read", async () => { + agentList = null; + try { + const graded = await grade([issue("i1")], [runOf("r1", "i1", 0, 10)], [record()], 0); + expect(graded.pass).toBe(false); + expect(graded.markdown).toContain("FAIL agent slot evidence"); + expect(events).toEqual(expect.arrayContaining(["hosted_agents_unavailable", "hosted_agent_slots_unavailable"])); + } finally { + agentList = agents; + } + }); +}); diff --git a/tests/restart-chaos/hosted-gate.ts b/tests/restart-chaos/hosted-gate.ts new file mode 100644 index 00000000000..3c880e5c384 --- /dev/null +++ b/tests/restart-chaos/hosted-gate.ts @@ -0,0 +1,123 @@ +import { agentSlotsFromRows, queueWaitFairnessFrom, type QueueWaitFairness } from "./agent-slots.js"; +import { gradeR1 } from "./grader.js"; +import type { ContinuityReport, DetectedRelease } from "./hosted-release.js"; +import { ms, type HostedRunRow } from "./hosted-readers.js"; +import type { HostedTaskRecord } from "./hosted-workload.js"; +import type { RetryClick, SwapMetrics } from "./release-swap-grade.js"; +import { stagingMixBudgetMs, SUBTASK_ROUNDS } from "./staging-mix.js"; +import type { FaultReceipt, RuleResult, RunSample, TaskRecord } from "./types.js"; + +const iso = (at: number | null) => at === null ? null : new Date(at).toISOString(); + +/** + * Only a release whose health probes failed had an outage. A release that two builds served without a failed probe + * gives R1 no outage credit, so a slow task after a clean swap still fails R1. + */ +export function releaseOutages(releases: readonly DetectedRelease[]): FaultReceipt[] { + return releases.filter(release => release.firstFailureAt !== null && release.firstFailureAt < release.firstNewAt).map((release): FaultReceipt => ({ + faultId: `release-${release.index}`, kind: "K15", requestedBy: "hosted-release", signal: "none", holdMs: 0, + requestedAt: release.firstFailureAt!, startedAt: release.firstFailureAt!, stoppedAt: null, restartedAt: null, + healthyAt: release.firstNewAt, readyAt: release.firstNewAt, phases: [], serverLogOffsetAtStart: 0, serverLogOffsetAtReady: null, error: null, + })); +} + +function runSample(run: HostedRunRow): RunSample { + return { id: run.runId, status: run.status, createdAt: iso(run.createdAt), startedAt: iso(run.startedAt), finishedAt: iso(run.finishedAt), + errorCode: run.errorCode, retryOfRunId: run.retryOfRunId, wakeCommentIds: run.wakeCommentIds, livenessState: null }; +} + +/** The task view fields R1 reads. */ +export interface HostedIssueView { status?: string | null; completedAt?: string | number | null } +/** The agent fields the queue-wait proof reads. */ +export interface HostedAgentRow { id: string; runtimeConfig?: { heartbeat?: { maxConcurrentRuns?: unknown } } | null } + +/** The fields of `TaskRecord` that `gradeR1` reads, so the hosted adapter's boundary is checked by the compiler. */ +type R1Subject = Pick + & { plan: Pick; retry: { at: number } | null }; + +/** + * The queue-wait proof for every run of the company, built once per grade. `missing` names the run's agents whose + * `maxConcurrentRuns` could not be read (an unreadable agent list or a restricted agent view), so no wait of theirs can + * be proven fair; the gate fails on it instead of grading R1 without the evidence. + */ +export function hostedQueueWait(agentIds: readonly string[], agents: readonly HostedAgentRow[], companyRuns: readonly HostedRunRow[]): + { queueWait: QueueWaitFairness; missing: string[] } { + const slots = agentSlotsFromRows(agents.map(agent => ({ id: agent.id, maxConcurrentRuns: agent.runtimeConfig?.heartbeat?.maxConcurrentRuns })), + companyRuns.map(run => ({ id: run.runId, agentId: run.agentId, createdAt: run.createdAt, invocationSource: run.invocationSource, + retryOfRunId: run.retryOfRunId, startedAt: run.startedAt, finishedAt: run.finishedAt }))); + const capped = new Set(slots.agents.filter(agent => agent.maxConcurrentRuns !== null).map(agent => agent.agentId)); + return { queueWait: queueWaitFairnessFrom(slots), missing: agentIds.filter(id => !capped.has(id)) }; +} + +/** + * The local waves' R1 for one hosted task: done within the staging-mix budget plus outage time, plus the first run's + * queue wait when the agent's slots prove it fair. The API has no wake evidence, so a follow-up run's wait is never + * credited (the local grader refuses it the same way). + */ +export function hostedR1(input: { record: HostedTaskRecord; view: HostedIssueView | undefined; runs: readonly HostedRunRow[]; + queueWait: QueueWaitFairness; outages: readonly FaultReceipt[] }): RuleResult { + const { record, view } = input; + const completedAt = ms(view?.completedAt); + const status = String(view?.status ?? record.status); + const restAt = status === "done" ? (completedAt ?? record.doneAt) : null; + const subject: R1Subject = { + plan: { workload: "SM", nonce: record.plan.nonce }, issueId: record.issueId, identifier: record.identifier, createdAt: record.createdAt, + launchError: record.createError, samples: [{ at: restAt ?? Date.now(), reachable: true, serverReady: true, runs: input.runs.map(runSample) }], + retry: record.retryClicks.length ? { at: record.retryClicks[0]!.at } : null, restAt, finalStatus: status, timedOut: false, + followUp: record.followUp ? { at: record.followUp.at, commentId: record.followUp.commentId, error: record.followUp.error } : null, + }; + // gradeR1 reads only the R1Subject fields; without runWakes it refuses a follow-up run's queue credit, as above. + return gradeR1(subject as unknown as TaskRecord, stagingMixBudgetMs(record.plan.task), input.outages, input.queueWait); +} + +/** A planned task's agent-created subtask graded on its own: created when the issue was, with the budget of its two rounds. */ +export function subtaskRecord(view: { id: string; identifier?: string | null; status?: string | null; createdAt?: string | number | null }, retryClicks: readonly RetryClick[]): HostedTaskRecord { + const created = ms(view.createdAt); + return { plan: { index: -1, adapter: "opencode", nonce: String(view.id), task: { turnMinutes: 0, rounds: SUBTASK_ROUNDS, subtasks: 0, followUpAfterMs: null, + arriveAtMs: 0, holdMs: 0 } }, issueId: String(view.id), identifier: view.identifier ? String(view.identifier) : null, + createdAt: created, createError: created === null ? "subtask has no createdAt" : null, + status: String(view.status), lastLiveAt: null, followUp: null, retryClicks: [...retryClicks], retriedRuns: [], doneAt: null }; +} + +export interface HostedGate { + pass: boolean; + r1Failed: Array<{ identifier: string | null; detail: string }>; + bannersShown: number; + healthFailures: number; + duplicates: number; + stopThenStart: number[]; + unloaded: number[]; + lines: string[]; +} + +/** + * Gate D: every task hands-off and inside R1, no Retry banner, no failed health probe, no duplicated work, and every + * release a swap two builds served (never stop-then-start) with at least one live run carried across it. A fixed batch + * (`--hosted-releases 0`) waits for no release, so it is not required to see one. + */ +export function hostedGate(input: { handsOffPass: boolean; wantedReleases: number; createErrors: number; slotEvidenceMissing: readonly string[]; r1: ReadonlyArray<{ identifier: string | null; result: RuleResult }>; bannersShown: number; + health: ContinuityReport; duplicates: { overlappingRuns: readonly unknown[]; duplicateComments: readonly unknown[]; duplicateChildren: readonly unknown[] }; + releases: readonly DetectedRelease[]; metrics: readonly SwapMetrics[] }): HostedGate { + const r1Failed = input.r1.filter(entry => entry.result.verdict === "fail").map(entry => ({ identifier: entry.identifier, detail: entry.result.detail })); + const duplicates = input.duplicates.overlappingRuns.length + input.duplicates.duplicateComments.length + input.duplicates.duplicateChildren.length; + const stopThenStart = input.releases.filter(release => release.shape === "stop_then_start").map(release => release.index); + const unloaded = input.metrics.filter(metric => metric.survivors.length === 0).map(metric => metric.index); + const checks: Array<[string, boolean, string]> = [ + ["hands-off", input.handsOffPass, input.handsOffPass ? "every started task done without a person" : "a task did not finish hands-off"], + ["R1", r1Failed.length === 0, r1Failed.length ? `${r1Failed.length} failed (${r1Failed.slice(0, 3).map(entry => `${entry.identifier ?? "?"}: ${entry.detail}`).join("; ")})` + : `${input.r1.filter(entry => entry.result.verdict === "pass").length} passed`], + ["agent slot evidence", input.slotEvidenceMissing.length === 0, input.slotEvidenceMissing.length + ? `maxConcurrentRuns unreadable for ${input.slotEvidenceMissing.length} agent(s), so R1 queue credit cannot be proven` : "read for every agent"], + ["tasks created", input.createErrors === 0, `${input.createErrors} task(s) could not be created`], + ["Retry banners", input.bannersShown === 0, `${input.bannersShown} shown`], + ["health probes", input.health.failures === 0, `${input.health.failures} of ${input.health.requests} failed`], + ["duplicates", duplicates === 0, `${duplicates} (overlapping runs, duplicated agent comments, duplicated subtasks)`], + ["swap shape", stopThenStart.length === 0, stopThenStart.length ? `stop-then-start at release ${stopThenStart.join(", ")}` : "no stop-then-start release"], + ...(input.wantedReleases > 0 ? [["releases under load", input.releases.length > 0 && unloaded.length === 0, + input.releases.length === 0 ? "no release detected" : unloaded.length ? `no live run across release ${unloaded.join(", ")}` + : `${input.releases.length} release(s), each with a live run`]] as Array<[string, boolean, string]> : []), + ]; + const pass = checks.every(([, ok]) => ok); + return { pass, r1Failed, bannersShown: input.bannersShown, healthFailures: input.health.failures, duplicates, stopThenStart, unloaded, + lines: [`Gate D: ${pass ? "PASS" : "FAIL"}`, ...checks.map(([name, ok, detail]) => `- ${ok ? "pass" : "FAIL"} ${name}: ${detail}`)] }; +} diff --git a/tests/restart-chaos/hosted-grade.ts b/tests/restart-chaos/hosted-grade.ts index 5fbcc6d3433..696cd8e2c45 100644 --- a/tests/restart-chaos/hosted-grade.ts +++ b/tests/restart-chaos/hosted-grade.ts @@ -3,7 +3,8 @@ import { writeFile } from "node:fs/promises"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { requestBreakdown, type HostedApi, type RequestRecord } from "./hosted-api.js"; -import { trackBanner, type RetryBanner } from "./hosted-workload.js"; +import { hostedGate, hostedQueueWait, hostedR1, releaseOutages, subtaskRecord, type HostedAgentRow } from "./hosted-gate.js"; +import { trackBanner, type HostedTaskRecord, type RetryBanner } from "./hosted-workload.js"; import { handsOffSummary, invariantLine, readAssignedWorkInvariant } from "./hosted-handsoff.js"; import { releaseMarkdown, type ContinuityReport, type DetectedRelease } from "./hosted-release.js"; import { @@ -39,6 +40,10 @@ export interface HostedGradeInput { now?: () => number; quietMs?: number; requests?: () => readonly RequestRecord[]; + /** The run's planned task records: R1 and the create check read them. */ + tasks: () => readonly HostedTaskRecord[]; + /** `--hosted-releases`: 0 is a fixed batch that waits for no release. */ + wantedReleases: number; } export function releaseWindows(releases: readonly DetectedRelease[]): SwapWindow[] { @@ -152,6 +157,22 @@ export async function gradeHosted(input: HostedGradeInput): Promise<{ headline: chain: await failureChain(api, run.runId) }); const health = input.health(); const apiContinuity = input.apiContinuity(); + const handsOffPass = result.uncollected === 0 && result.stillRunning === 0 && result.started > 0 && result.completedWithoutHuman === result.started; + const agents = (await api.get(`/api/companies/${companyId}/agents`).catch(error => { + input.record("hosted_agents_unavailable", { error: String(error) }); + return [] as HostedAgentRow[]; + })).filter(agent => agentIds.includes(String(agent.id))); + const slotProof = hostedQueueWait(agentIds, agents, runs); + if (slotProof.missing.length) input.record("hosted_agent_slots_unavailable", { agentIds: slotProof.missing }); + const outages = releaseOutages(input.releases); + const started = new Set(issues.map(issue => issue.id)); + const planned = input.tasks(); + const plannedIds = new Set(planned.flatMap(task => task.issueId ? [task.issueId] : [])); + const subtasks = issues.filter(issue => !plannedIds.has(issue.id) && issue.parentId !== null && plannedIds.has(issue.parentId)).map(issue => subtaskRecord({ ...views.get(issue.id), id: issue.id, + identifier: issue.identifier }, clicks.get(issue.id) ?? [])); + const r1 = [...planned.filter(task => task.issueId && started.has(task.issueId)), ...subtasks].map(task => ({ identifier: task.identifier, + result: hostedR1({ record: task, view: views.get(task.issueId!), runs: runsByIssue.get(task.issueId!) ?? [], queueWait: slotProof.queueWait, outages }) })); + const gate = hostedGate({ handsOffPass, wantedReleases: input.wantedReleases, slotEvidenceMissing: slotProof.missing, createErrors: planned.filter(task => task.createError).length, r1, bannersShown: handsOff.bannerShown, health, duplicates, releases: input.releases, metrics }); const releaseSpans = input.releases.map(release => ({ from: release.lastOldOnlyAt, to: Math.max(release.firstNewAt, release.lastOldAt) + 60_000 })); const records = input.requests?.() ?? []; const breakdown = { api: requestBreakdown(records.filter(record => record.kind === "api"), releaseSpans), @@ -162,6 +183,8 @@ export async function gradeHosted(input: HostedGradeInput): Promise<{ headline: ...handsOff.lines, invariantLine(invariant), "", + ...gate.lines, + "", releaseSwapMarkdown(result, metrics, windows.map(() => null), duplicates, grades, failedOutsideSwaps(windows, facts)).trimEnd(), "", "Hosted mode: releases came from the instance's own deploys, detected by polling /api/health. Lost-while-old-alive and late/preparation probes are not measured (see Unavailable).", @@ -172,12 +195,12 @@ export async function gradeHosted(input: HostedGradeInput): Promise<{ headline: ...UNAVAILABLE_OVER_API.map(item => `- ${item}`), "", ].join("\n"); - await writeFile(path.join(input.dir, "hosted-release-swap.json"), `${JSON.stringify({ schema: "paperclip.restart-chaos.hosted-release-swap/v1", result, handsOff: { ...handsOff, banners: Object.fromEntries(banners) }, invariant, + await writeFile(path.join(input.dir, "hosted-release-swap.json"), `${JSON.stringify({ schema: "paperclip.restart-chaos.hosted-release-swap/v1", result, handsOff: { ...handsOff, banners: Object.fromEntries(banners) }, gate, r1, invariant, releases: input.releases, health, api: apiContinuity, requestBreakdown: breakdown, metrics, duplicates, failedRuns, grades, drained, unavailable: UNAVAILABLE_OVER_API }, null, 2)}\n`, { mode: 0o600 }); await writeFile(path.join(input.dir, "SUMMARY.md"), markdown, { mode: 0o600 }); - input.record("hosted_graded", { headline: result.headline, byVerdict: result.byVerdict, byLabel: result.byLabel, retry: result.retry, + input.record("hosted_graded", { gate: gate.pass, r1Failed: gate.r1Failed.length, headline: result.headline, byVerdict: result.byVerdict, byLabel: result.byLabel, retry: result.retry, failedPerRelease: metrics.map(metric => metric.failedRuns.length) }); return { headline: handsOff.lines[0]!, markdown, - pass: result.uncollected === 0 && result.stillRunning === 0 && result.started > 0 && result.completedWithoutHuman === result.started }; + pass: gate.pass }; } diff --git a/tests/restart-chaos/hosted-main.test.ts b/tests/restart-chaos/hosted-main.test.ts index 1322dfd8aa7..e58b3874c8d 100644 --- a/tests/restart-chaos/hosted-main.test.ts +++ b/tests/restart-chaos/hosted-main.test.ts @@ -38,7 +38,7 @@ describe("hosted run control", () => { let stopped = false; const graded = await gradeHosted({ api, companyId: "c1", agentIds: ["a1"], releases: [], health: () => continuity([]), apiContinuity: () => continuity([]), stopObserver: async () => { stopped = true; record.retryClicks.push({ at: 0, accepted: true, via: "observer" }); }, - observedClicks: () => observedClicks([record]), lastReleaseAt: Date.now() - 1_000, settleMs: 5, drainMs: 5, dir, cancelled: () => false, record: () => {}, pollMs: 5 }); + observedClicks: () => observedClicks([record]), lastReleaseAt: Date.now() - 1_000, settleMs: 5, drainMs: 5, dir, cancelled: () => false, record: () => {}, pollMs: 5, tasks: () => [], wantedReleases: 0 }); expect(stopped).toBe(true); expect(graded.headline).toBe("0 / 1 tasks completed hands-off"); }); diff --git a/tests/restart-chaos/hosted-main.ts b/tests/restart-chaos/hosted-main.ts index 3053ec27b5a..47b20922200 100644 --- a/tests/restart-chaos/hosted-main.ts +++ b/tests/restart-chaos/hosted-main.ts @@ -164,6 +164,7 @@ async function hostedSession(input: { options: HostedOptions; dir: string; timel health: () => continuity(probe.samples), apiContinuity: () => continuity(requestSamples(apiRequests(api.requests))), requests: () => api.requests, quietMs: options.quietMs, stopObserver: () => polling!.stop(), observedClicks: () => observedClicks(workload.records), banners: () => new Map(workload.banners), + tasks: () => workload.records, wantedReleases: options.releases, agentNames: Object.values(state.agents).map(agent => agent!.name), lastReleaseAt: releases.at(-1)?.lastOldAt ?? Date.now(), settleMs: options.settleMs, drainMs: options.drainMs, dir, cancelled, record }); console.info(`\n${graded.markdown}`); diff --git a/tests/restart-chaos/hosted-readers.test.ts b/tests/restart-chaos/hosted-readers.test.ts index c23f240dac5..5c6ef13000e 100644 --- a/tests/restart-chaos/hosted-readers.test.ts +++ b/tests/restart-chaos/hosted-readers.test.ts @@ -104,7 +104,7 @@ describe("hosted API readers", () => { let latestApi = { requests: 0, failures: 0, slow: 0, longestGapMs: 0, longestGap: null as { from: number; to: number } | null }; const graded = await gradeHosted({ api, companyId: "c1", agentIds: ["a1"], releases, health: () => ({ requests: 2, failures: 0, slow: 0, longestGapMs: 1_000, longestGap: null }), apiContinuity: () => latestApi, stopObserver: async () => { latestApi = { requests: 9, failures: 3, slow: 1, longestGapMs: 4_000, longestGap: { from: 0, to: 4_000 } }; }, observedClicks: () => new Map(), lastReleaseAt: Date.now() - 10_000, - settleMs: 5, drainMs: 5, dir, cancelled: () => false, record: () => {}, pollMs: 5 }); + settleMs: 5, drainMs: 5, dir, cancelled: () => false, record: () => {}, pollMs: 5, tasks: () => [], wantedReleases: releases.length }); expect(writes.filter(write => write.route.endsWith("/retry"))).toEqual([{ method: "POST", route: "/api/issues/i2/retry", body: { idempotencyKey: expect.any(String), observedRunId: "r2" } }]); expect(graded.headline).toBe("1 / 3 tasks completed hands-off"); expect(graded.markdown.split("\n").slice(0, 4)).toEqual(["1 / 3 tasks completed hands-off", "Retry banner shown: 1 (cleared by itself: 0)", diff --git a/tests/restart-chaos/hosted-readers.ts b/tests/restart-chaos/hosted-readers.ts index 1e54e25435d..6d7cb804618 100644 --- a/tests/restart-chaos/hosted-readers.ts +++ b/tests/restart-chaos/hosted-readers.ts @@ -1,3 +1,4 @@ +import { wakeInputCommentIds } from "./follow-up-delivery.js"; import type { HostedApi } from "./hosted-api.js"; import { isAutoNotice, type GradeRun, type GradeTaskInput } from "./release-swap-grade.js"; @@ -24,6 +25,9 @@ export interface HostedRunRow { finishedAt: number | null; lastActivityAt: number | null; ownerPid: null; + invocationSource: string | null; + retryOfRunId: string | null; + wakeCommentIds: string[]; } export interface HostedIssueRow { @@ -36,7 +40,7 @@ export interface HostedIssueRow { hidden: boolean; } -const ms = (value: unknown): number | null => { +export const ms = (value: unknown): number | null => { if (value === null || value === undefined || value === "") return null; const parsed = typeof value === "number" ? value : Date.parse(String(value)); return Number.isFinite(parsed) ? parsed : null; @@ -50,6 +54,7 @@ export function runRow(run: Json): HostedRunRow { errorCode: text(run.errorCode), error: run.error ? String(run.error).slice(0, 400) : null, createdAt: ms(run.createdAt), startedAt: ms(run.startedAt), finishedAt: ms(run.finishedAt), lastActivityAt: activity.length ? Math.max(...activity) : null, ownerPid: null, + invocationSource: text(run.invocationSource), retryOfRunId: text(run.retryOfRunId), wakeCommentIds: wakeInputCommentIds(run, undefined), }; }