From 278660149ca69f0b993195e21c34c5815ec70689 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Mon, 16 Nov 2026 01:13:19 -0600 Subject: [PATCH] test(recovery): bound every wait in the agent-row census and stop it starting real runs (#21867) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The recovery census tests prove that recovery queues do not re-check settled tasks without a reason > - The agent-row census from #21861 stopped producing output in a server test shard and ran until the 20-minute job limit > - Its tests let real agent runs start while `setTimeout` was faked, and no wait in the file had a bound (the test timeout was 30 minutes) > - This pull request stops the census from starting real runs, bounds every wait with a real timer, and makes the file much faster > - The benefit is that the file cannot hang a shard, and a stuck step fails in seconds with a message that names the step ## Linked Issues or Issue Description Refs #21861 **What happened?** In a 10-shard server test run, `server/src/__tests__/issue-recovery-agent-row-write-census.test.ts` finished its third test and then wrote no more output until the job reached its 20-minute limit. The file had a 30-minute timeout per test, so the hang never failed as a test. Several tests let real agent runs start while `setTimeout` was faked: - the idle-agent guard marked a live run succeeded while its task stayed in progress, so recovery continued the task; - the hold-release guard released an agent that had a `todo` task; - the pause and terminate guards stopped agents with live native runs. A run execution that waits on a faked timer, or holds a lock that a delivery pass then waits on, does not end. A local run of the old file showed 4 run executions ("heartbeat execution setup failed"). **Expected behavior** The census never starts a real run. Every wait has a bound and a message that names what it waited for. The file finishes well under 60 seconds. **Steps to reproduce** 1. Run the old file and search its output for "heartbeat execution setup failed": 4 run executions. 2. Run it in a slow, shared CI shard: the file can stop with no output until the job limit. **Paperclip version or commit** `master` at 94cb7ad359. ## What Changed - Fixture agents take no on-demand wakes (`wakeOnDemand: false`), so a recovery check cannot dispatch a run. The census counts queue checks, so this does not change what the tests measure. - The idle-agent guard writes the configuration of an agent that never ran, instead of ending the second agent's live run. - The held agent's waiting task is a settled review, not a `todo` task. - The pause and terminate guards use two idle agents without live runs. - Each step (run-event tick, fake-timer advance, pass settlement, writes, lifecycle commands) has a real 20-second bound that names what it waited for. The bound uses the real `setTimeout`, captured before the timers are faked. Each test has a 2-minute timeout instead of 30 minutes. - The fixture is smaller (140 + 20 settled reviews instead of 700 + 80). Warm-up is 6 simulated minutes instead of 12, and each write window is 6 minutes instead of 7. - The full-pass guard uses a 15-minute full pass (`fullBackstopMs`) instead of the hourly default. The fixture's `start` accepts `fullBackstopMs` for this. ## Verification - Old file, 4 copies at the same time (local CPU contention): all passed, 190–211 s per file, 172 s of test time, 4 real run executions. - New file, 4 copies at the same time: all passed, 48.5–49.1 s per file (about 30 s of test time; each test 4–6 s), 0 run executions. - The census still fails without the #21861 fix. With `notifyRunDispatch` put back in `agent-configuration.ts`, the census test fails with 480 settled checks (three whole-company reloads of 160) and 519 `company` checks. With the fix, it has 0 settled checks. - Server `tsc --noEmit` passed. ## Risks - Test-only change. No product code changed. - The guards now use idle agents and settled tasks rather than agents with live runs, so they do not cover a pause or termination that must stop a live run. Those flows have their own lifecycle tests. The live-run census (`issue-recovery-live-run-agent-census.test.ts`) still covers the hourly full pass with the real interval. - The fixture is smaller, but the census still compares whole-company reloads against zero. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- ...ue-recovery-agent-row-write-census.test.ts | 156 ++++++++++-------- .../__tests__/issue-recovery.test-fixture.ts | 2 +- 2 files changed, 85 insertions(+), 73 deletions(-) diff --git a/server/src/__tests__/issue-recovery-agent-row-write-census.test.ts b/server/src/__tests__/issue-recovery-agent-row-write-census.test.ts index 7104f1a74fc..450b0f125bd 100644 --- a/server/src/__tests__/issue-recovery-agent-row-write-census.test.ts +++ b/server/src/__tests__/issue-recovery-agent-row-write-census.test.ts @@ -3,8 +3,7 @@ import { eq } from "drizzle-orm"; import { expect, it, vi } from "vitest"; import { agents, heartbeatRuns, issues } from "@paperclipai/db"; import { workSignalDebug } from "@paperclipai/db/work-signal-debug"; -import { ISSUE_RECOVERY_BACKSTOP_MS, ISSUE_RECOVERY_FULL_BACKSTOP_MS, notifyIssueGraphRecovery } - from "../services/issue-recovery-notifications.js"; +import { ISSUE_RECOVERY_BACKSTOP_MS, notifyIssueGraphRecovery } from "../services/issue-recovery-notifications.js"; import { agentService } from "../services/agents.js"; import { createAgentLifecycle, createAgentLifecycleEffects } from "../services/agent-lifecycle.js"; import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js"; @@ -13,9 +12,22 @@ vi.hoisted(() => { process.env.PAPERCLIP_DEBUG_WORK_SIGNALS = "stranded-assignme const MINUTE = 60_000; const HOUR = 60 * MINUTE; +const TEST_TIMEOUT_MS = 120_000; +const STEP_TIMEOUT_MS = 20_000; +const FULL_PASS_MS = 15 * MINUTE; const TOPICS = new Set(["stranded-assignments", "assigned-work-invariant"]); +const realSetTimeout = globalThis.setTimeout; +const realClearTimeout = globalThis.clearTimeout; type Recorded = { at: number; topic: string; name: string; fields: Record }; +function bounded(work: Promise, waitingFor: () => string): Promise { + let timer: ReturnType | undefined; + const expired = new Promise((_, reject) => { + timer = realSetTimeout(() => reject(new Error(`Timed out after ${STEP_TIMEOUT_MS} ms waiting for ${waitingFor()}`)), STEP_TIMEOUT_MS); + }); + return Promise.race([work, expired]).finally(() => realClearTimeout(timer)); +} + describeIssueRecovery("agent row writes during a long run (census)", () => { const state = issueRecoveryFixture({ meter: true }); @@ -23,9 +35,14 @@ describeIssueRecovery("agent row writes during a long run (census)", () => { const until = Date.now() + ms; let nextTick = Date.now(); while (Date.now() < until) { - if (onTick && Date.now() >= nextTick) { await onTick(); nextTick = Date.now() + 10_000; } - await vi.advanceTimersByTimeAsync(Math.min(1000, until - Date.now())); - await Promise.allSettled([...state.passes]); await immediate(); + const at = new Date(Date.now()).toISOString(); + if (onTick && Date.now() >= nextTick) { + await bounded(onTick(), () => `the run event at simulated ${at} to commit`); + nextTick = Date.now() + 10_000; + } + await bounded(vi.advanceTimersByTimeAsync(Math.min(1000, until - Date.now())), () => `simulated time to advance past ${at}`); + await bounded(Promise.allSettled([...state.passes]), () => `${state.passes.size} delivery passes started by simulated ${at} to settle`); + await immediate(); } } @@ -49,11 +66,14 @@ describeIssueRecovery("agent row writes during a long run (census)", () => { async function seed() { const f = await state.fixture(); const config = { adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "0"], cwd: process.cwd() }, - runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: true } }, role: "engineer" }; + runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: false } }, role: "engineer" }; const old = new Date(Date.now() - 2 * HOUR); - const [big, second, held] = await state.db.insert(agents).values([ - { ...config, companyId: f.companyId, name: "Big worker", status: "idle", createdAt: old, updatedAt: old, lastHeartbeatAt: old }, - { ...config, companyId: f.companyId, name: "Second worker", status: "idle", createdAt: old, updatedAt: old, lastHeartbeatAt: old }, + const idle = { ...config, companyId: f.companyId, status: "idle", createdAt: old, updatedAt: old, lastHeartbeatAt: old }; + const [big, second, spare, other, held] = await state.db.insert(agents).values([ + { ...idle, name: "Big worker" }, + { ...idle, name: "Second worker" }, + { ...idle, name: "Spare worker" }, + { ...idle, name: "Other worker" }, { ...config, companyId: f.companyId, name: "Held worker", status: "paused", lifecycleHolds: ["manual"], pauseReason: "manual", createdAt: old, updatedAt: old }, ]).returning({ id: agents.id }); @@ -66,22 +86,24 @@ describeIssueRecovery("agent row writes during a long run (census)", () => { runtimeMode: "native", startedAt: finishedAt, finishedAt, updatedAt: finishedAt, contextSnapshot: { issueId: row.id } }))); return rows.map(row => row.id); }; - const bigTasks = await reviews(big!.id, 700); - const secondTasks = await reviews(second!.id, 80); - const [bigLive, secondLive, waiting] = await state.db.insert(issues).values([ + const bigTasks = await reviews(big!.id, 140); + const secondTasks = await reviews(second!.id, 20); + const spareTasks = await reviews(spare!.id, 5); + const otherTasks = await reviews(other!.id, 5); + const [waiting] = await reviews(held!.id, 1); + const [bigLive, secondLive] = await state.db.insert(issues).values([ { companyId: f.companyId, title: "Big live task", status: "in_progress", assigneeAgentId: big!.id }, { companyId: f.companyId, title: "Second live task", status: "in_progress", assigneeAgentId: second!.id }, - { companyId: f.companyId, title: "Waits for its agent", status: "todo", assigneeAgentId: held!.id, createdAt: old, updatedAt: old }, ]).returning({ id: issues.id }); - return { ...f, bigId: big!.id, secondId: second!.id, heldId: held!.id, bigTasks, secondTasks, - bigLiveTask: bigLive!.id, secondLiveTask: secondLive!.id, waitingTask: waiting!.id }; + return { ...f, bigId: big!.id, secondId: second!.id, spareId: spare!.id, otherId: other!.id, heldId: held!.id, + bigTasks, secondTasks, spareTasks, otherTasks, waitingTask: waiting!, bigLiveTask: bigLive!.id, secondLiveTask: secondLive!.id }; } - async function boot() { - const f = await seed(); + async function boot(options: { fullBackstopMs?: number } = {}) { + const f = await bounded(seed(), () => "the fixture to be seeded"); const events = record(); vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); - await state.start({}); + await bounded(state.start(options), () => "the recovery workers to start"); const startRun = async (agentId: string, issueId: string) => { const [run] = await state.db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId, status: "running", runtimeMode: "native", startedAt: new Date(), contextSnapshot: { issueId }, nativeIssueId: issueId }) @@ -90,13 +112,13 @@ describeIssueRecovery("agent row writes during a long run (census)", () => { await state.db.update(agents).set({ status: "running", updatedAt: new Date() }).where(eq(agents.id, agentId)); return run!.id; }; - const bigRun = await startRun(f.bigId, f.bigLiveTask); - await startRun(f.secondId, f.secondLiveTask); + const bigRun = await bounded(startRun(f.bigId, f.bigLiveTask), () => "the big agent's live run to be recorded"); + await bounded(startRun(f.secondId, f.secondLiveTask), () => "the second agent's live run to be recorded"); const tick = () => state.db.transaction(async tx => { await tx.update(heartbeatRuns).set({ updatedAt: new Date() }).where(eq(heartbeatRuns.id, bigRun)); await notifyIssueGraphRecovery(tx, { companyId: f.companyId, runId: bigRun }); }); - await advance(12 * MINUTE, tick); + await advance(ISSUE_RECOVERY_BACKSTOP_MS + MINUTE, tick); events.length = 0; return { ...f, events, bigRun, tick }; } @@ -104,13 +126,20 @@ describeIssueRecovery("agent row writes during a long run (census)", () => { const checksOf = (events: Recorded[], issueId: string) => events.filter(event => event.topic === "stranded-assignments" && event.name === "check" && event.fields.issueId === issueId); + async function expectCheckedWithin(f: Awaited>, issueId: string, changedAt: number, boundMs: number) { + await advance(boundMs, f.tick); + const [checked] = checksOf(f.events, issueId).filter(event => event.at >= changedAt); + expect(checked, `task ${issueId} was not checked within ${boundMs / MINUTE} simulated minutes`).toBeDefined(); + expect(checked!.at - changedAt).toBeLessThanOrEqual(boundMs); + } + it("does not re-check either running agent's settled tasks when their rows are written during the runs", async () => { const f = await boot(); const svc = agentService(state.db); const before = state.meter.read(); const writes: Array<[string, () => Promise]> = [ ["big agent runtime configuration", () => svc.update(f.bigId, - { runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: true }, revision: 1 } })], + { runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: false }, revision: 1 } })], ["second agent adapter configuration", () => svc.update(f.secondId, { adapterConfig: { command: process.execPath, args: ["-e", "1"], cwd: process.cwd() } })], ["big agent execution start", () => state.db.update(agents).set({ status: "running", updatedAt: new Date() }) @@ -119,89 +148,72 @@ describeIssueRecovery("agent row writes during a long run (census)", () => { ["big agent adapter configuration", () => svc.update(f.bigId, { adapterConfig: { command: process.execPath, args: ["-e", "2"], cwd: process.cwd() } })], ]; - for (const [, write] of writes) { - await write(); - await advance(7 * MINUTE, f.tick); + const minutesPerWrite = 6; + for (const [label, write] of writes) { + await bounded(write(), () => `the ${label} write to commit`); + await advance(minutesPerWrite * MINUTE, f.tick); } const after = state.meter.read(); const settled = new Set([...f.bigTasks, ...f.secondTasks]); const settledChecks = f.events.filter(event => event.name === "check" && settled.has(String(event.fields.issueId))).length; - const result = { writes: writes.map(([label]) => label), minutes: writes.length * 7, settledChecks, byWake: byWake(f.events), - statements: after.statements - before.statements }; + const result = { writes: writes.map(([label]) => label), minutes: writes.length * minutesPerWrite, settledChecks, + byWake: byWake(f.events), statements: after.statements - before.statements }; console.info(`agent-row census: ${JSON.stringify(result)}`); expect(settledChecks).toBe(0); expect(result.byWake["stranded-assignments"]?.company ?? 0).toBe(0); expect(result.byWake["stranded-assignments"]?.reload ?? 0).toBe(0); - }, 1_800_000); + }, TEST_TIMEOUT_MS); it("still checks a running agent's task within the backstop bound when it becomes stranded after a configuration write", async () => { const f = await boot(); - await agentService(state.db).update(f.bigId, { runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: true }, revision: 2 } }); + await bounded(agentService(state.db).update(f.bigId, { runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: false }, revision: 2 } }), + () => "the big agent configuration write to commit"); await advance(MINUTE, f.tick); f.events.length = 0; const target = f.bigTasks[17]!; const changedAt = Date.now(); - await state.db.update(issues).set({ status: "todo", updatedAt: new Date() }).where(eq(issues.id, target)); - await advance(ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE, f.tick); - const [checked] = checksOf(f.events, target); - expect(checked).toBeDefined(); - expect(checked!.at - changedAt).toBeLessThanOrEqual(ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE); - }, 1_800_000); + await bounded(state.db.update(issues).set({ status: "todo", updatedAt: new Date() }).where(eq(issues.id, target)).then(() => undefined), + () => "the task write without a hint to commit"); + await expectCheckedWithin(f, target, changedAt, ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE); + }, TEST_TIMEOUT_MS); it("still checks an idle agent's tasks within the backstop bound after its configuration is written", async () => { const f = await boot(); - await state.db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date(), updatedAt: new Date() }) - .where(eq(heartbeatRuns.agentId, f.secondId)); - await state.db.update(agents).set({ status: "idle", lastHeartbeatAt: new Date(), updatedAt: new Date() }) - .where(eq(agents.id, f.secondId)); - await advance(11 * MINUTE, f.tick); - f.events.length = 0; const changedAt = Date.now(); - await agentService(state.db).update(f.secondId, { adapterConfig: { command: process.execPath, args: ["-e", "3"], cwd: process.cwd() } }); - await advance(ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE, f.tick); - const [checked] = checksOf(f.events, f.secondTasks[5]!); - expect(checked).toBeDefined(); - expect(checked!.at - changedAt).toBeLessThanOrEqual(ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE); - }, 1_800_000); + await bounded(agentService(state.db).update(f.spareId, { adapterConfig: { command: process.execPath, args: ["-e", "3"], cwd: process.cwd() } }), + () => "the idle agent configuration write to commit"); + await expectCheckedWithin(f, f.spareTasks[2]!, changedAt, ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE); + }, TEST_TIMEOUT_MS); it("still checks a held agent's waiting task promptly when its hold is released", async () => { const f = await boot(); const changedAt = Date.now(); - await state.db.transaction(async tx => { + await bounded(state.db.transaction(async tx => { await createAgentLifecycleEffects().beforeStatusChange(tx as never, f.companyId, f.heldId, "paused", "idle"); await tx.update(agents).set({ status: "idle", lifecycleHolds: [], pauseReason: null, pausedAt: null, updatedAt: new Date() }) .where(eq(agents.id, f.heldId)); - }); - await advance(2 * MINUTE, f.tick); - const [checked] = checksOf(f.events, f.waitingTask); - expect(checked).toBeDefined(); - expect(checked!.at - changedAt).toBeLessThanOrEqual(MINUTE); - }, 1_800_000); + }), () => "the hold release to commit"); + await expectCheckedWithin(f, f.waitingTask, changedAt, MINUTE); + }, TEST_TIMEOUT_MS); - it("still checks a running agent's tasks promptly when the agent is paused or terminated", async () => { + it("still checks an agent's tasks promptly when the agent is paused or terminated", async () => { const f = await boot(); const lifecycle = createAgentLifecycle(state.db); let changedAt = Date.now(); - await lifecycle.pauseAgent(f.secondId); - await advance(2 * MINUTE, f.tick); - let [checked] = checksOf(f.events, f.secondTasks[3]!); - expect(checked).toBeDefined(); - expect(checked!.at - changedAt).toBeLessThanOrEqual(MINUTE); - f.events.length = 0; + await bounded(lifecycle.pauseAgent(f.spareId), () => "the pause to commit"); + await expectCheckedWithin(f, f.spareTasks[3]!, changedAt, MINUTE); changedAt = Date.now(); - await lifecycle.terminateAgent(f.bigId); - await advance(2 * MINUTE, f.tick); - [checked] = checksOf(f.events, f.bigTasks[9]!); - expect(checked).toBeDefined(); - expect(checked!.at - changedAt).toBeLessThanOrEqual(MINUTE); - }, 1_800_000); + await bounded(lifecycle.terminateAgent(f.otherId), () => "the termination to commit"); + await expectCheckedWithin(f, f.otherTasks[1]!, changedAt, MINUTE); + }, TEST_TIMEOUT_MS); - it("still re-checks every settled task in the hourly full pass after configuration writes", async () => { - const f = await boot(); - await agentService(state.db).update(f.bigId, { runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: true }, revision: 3 } }); - await advance(ISSUE_RECOVERY_FULL_BACKSTOP_MS + 5 * MINUTE, f.tick); + it("still re-checks every settled task in the next full pass after configuration writes", async () => { + const f = await boot({ fullBackstopMs: FULL_PASS_MS }); + await bounded(agentService(state.db).update(f.bigId, { runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: false }, revision: 3 } }), + () => "the big agent configuration write to commit"); + await advance(FULL_PASS_MS, f.tick); const checked = new Set(f.events.filter(event => event.topic === "stranded-assignments" && event.name === "check") .map(event => String(event.fields.issueId))); expect([...f.bigTasks, ...f.secondTasks].filter(id => !checked.has(id))).toEqual([]); - }, 1_800_000); + }, TEST_TIMEOUT_MS); }); diff --git a/server/src/__tests__/issue-recovery.test-fixture.ts b/server/src/__tests__/issue-recovery.test-fixture.ts index 4aa0b6ac764..7421fc349fb 100644 --- a/server/src/__tests__/issue-recovery.test-fixture.ts +++ b/server/src/__tests__/issue-recovery.test-fixture.ts @@ -53,7 +53,7 @@ export function issueRecoveryFixture(options: { meter?: boolean } = {}) { expect(errors).toEqual([]); }); afterAll(async () => { meter?.uninstall(); await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); - async function start(options: { backstopMs?: number } = { backstopMs: 0 }) { + async function start(options: { backstopMs?: number; fullBackstopMs?: number } = { backstopMs: 0 }) { coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, onError: error => errors.push(error) }); tasks.clear();