From 5c5a2fcebef7d6e270192976df00993b56cb8fc7 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 13 Nov 2026 08:51:33 -0600 Subject: [PATCH] fix(dispatch): start the assigned-work sweep's continuations and count only system recovery for the recovery share (#21391) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The heartbeat queue starts an agent's queued runs. #21234, #21277 and #21349 order them: person work first, a recovery share, aging, and a share for the longest-waiting run > - On hosted staging, one agent had about 250 queued runs. For 90 minutes after #21349 deployed, every run that started was a retry or continuation with `retryOfRunId`. The oldest queued runs, from 15 hours earlier, never started > - The dispatch order was not bypassed. The oldest-run share offered those runs, but the claim refused them every time, without a log at info level or a visible state > - The refusal came from the chat-control recovery gate. It needs `retryOfRunId` to trace an automatic continuation. The assigned-work invariant sweep queues continuations that name their source only in context, so the gate returned "unresolved", and an unresolved claim stays queued forever > - This pull request lets the gate check such a continuation against the task's previous run, and makes the recovery share count only system recovery starts > - The benefit is that the sweep's continuations start (it already told people "you don't need to press Retry"), and system recovery gets its share even when a person's recovery restarts keep starting ## Linked Issues or Issue Description Refs #21234, #21277, #21324, #21349, #21242 **What happened?** On hosted staging (build e23549cd9c), one agent with `maxConcurrentRuns` 20 had about 250 queued runs. From 12:06Z to 13:30Z, all 30 runs that started had `retryOfRunId` set. The oldest queued runs were automatic continuations (`issue_continuation_needed`) created from 22:53Z the day before. Each task's activity shows "Continued automatically after an interruption ... you don't need to press Retry" from the assigned-work invariant sweep, at the time the run was queued. None of these runs started. Assignments created at 07:51Z and later did start, in creation order. Those starts came through the oldest-run share after the share passed over the refused continuations. **Why did it happen?** 1. `assignedWorkSweep` queues the continuation with `previousRunId` in its context and no `retryOfRunId`. Its wake is a system wake (`requestedByActorId` null) with reason `issue_continuation_needed`. 2. At claim, `withChatControlRecoveryGate` calls `readChatControlRecoveryStop`. For an automatic continuation without `retryOfRunId`, it returned `unresolved`. At the claim stage, `unresolved` returns null: the run stays queued and nothing changes. So every dispatch pass offered these runs and every claim refused them. Retries and continuations with `retryOfRunId` pass the gate, so they were the only recovery work that started. 3. Separately, the recovery share (#21234) read the agent's recent starts and counted every start with a recovery wake reason as a recovery start. A person's `issue_recovery_action_restored` restart ranks as person work in the dispatcher, but it still reset the share. So the recovery share almost never fired while those restarts kept starting. **Free slots.** The "14 of 20 running" figure came from a capped run list. A full read at 14:03Z showed 22 runs `running` on that agent, so there were no free slots. The agent was over its limit, not under it. Overlapping dispatch passes after the 30 s stale agent start lock are a likely cause (each pass re-tried every refused run, so passes were slow). This pull request removes the refused runs that made passes slow. #21242 changes the claim-then-execute timing. **Expected behavior** An automatic continuation that the system queued starts in its turn. A chat close of the task's previous run still stops it. The recovery share goes to system recovery after three starts that were not system recovery. **Steps to reproduce** 1. Give an agent one slot. Interrupt a run on an in-progress task so that the assigned-work invariant finds a violation. 2. Run the assigned-work check. It queues an `issue_continuation_needed` continuation. 3. Queue fresher recovery runs on other tasks and dispatch. 4. See that the continuation stays queued on every pass. ## What Changed - `server/src/services/chat-control-recovery-stop.ts`: for an automatic continuation that has no `retryOfRunId`, the gate now walks to the task's latest earlier run (by `issueId`, `nativeIssueId` or the context task id, in the same company) and checks that run. If the task has no earlier run, the result is `clear`. A continuation with `retryOfRunId` is unchanged. A chat close of the previous run still stops the continuation. - `server/src/services/heartbeat/queue.ts`: the recovery-share counter reads which recent recovery-reason starts were person-originated, with the same source-chain query that the dispatcher already uses (`listPersonOriginSourceRunIds`, at most 4 ids). Only system recovery starts reset the share. This matches the in-pass rule (`isSystemRecoveryRun`). ## Verification - Red/green, embedded Postgres, real heartbeat queue (`server/src/__tests__/recovery-wake-herd.test.ts`): - "starts the assigned-work sweep's continuation of a task whose run was interrupted, ahead of fresher retries": on master the continuation stays queued for 60 s of dispatch passes; with the fix it starts. - "gives system recovery its share when the agent's recent starts were a person's recovery restarts": on master a person run starts first; with the fix the system recovery run starts. - `server/src/__tests__/heartbeat-process-recovery.test.ts`: "checks an automatic continuation that names no retry source against the task's previous run" proves that a chat close of the previous run still stops the continuation (`stopped`) and that it is `clear` when the close is not published. - `node server/scripts/typecheck-layers.mjs`: passes. - Related suites: `recovery-wake-herd`, `queued-run-dispatch-order`, `assigned-work-invariant`, `run-authority-read-projections`, `chat-run-milestone-work`, `legacy-preparation-retry-hold`, `explicit-native-continuation`, `saved-work-retry-eligibility`, `native-failed-turn-continuation.integration`, `heartbeat-process-recovery` pass, except: - `heartbeat-process-recovery` "schedules bounded retries for failed accepted interaction continuation wakes" is flaky on master too (1 of 3 runs failed on master source, same line). - `recovery-wake-herd` "delivers the continuation after a restart..." failed once with a hook timeout under parallel load and passes alone. - `runtime-selection.test.ts` fails locally in a nested worktree because a bundled skill file path does not exist there (environment, not this change). - After deploy: the queued `issue_continuation_needed` runs on the affected staging agent should start within the oldest-run share, and no run should stay queued longer than the aging bound plus a few starts. ## Risks - Behavior change: an automatic continuation without `retryOfRunId` was refused at claim and failed at dispatch with `chat_control_recovery_proof_unresolved`. It now inherits the chat-close check of the task's previous run instead. A chat close still stops it. - Already-queued stuck continuations on hosted instances will start after deploy. On the staging agent that is about 30 runs. They go through the normal order and shares, so they do not take all slots at once. - One extra indexed read per gate check of this shape, and one small recursive query per dispatch pass when recovery runs are queued. No schema change. Mixed versions are safe. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, in Claude Code with tool use (shell, file edit, built-in browser for read-only staging API reads). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [ ] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- .../heartbeat-process-recovery.test.ts | 19 +++++++ .../src/__tests__/recovery-wake-herd.test.ts | 56 +++++++++++++++++++ .../services/chat-control-recovery-stop.ts | 28 +++++++++- server/src/services/heartbeat/queue.ts | 7 ++- 4 files changed, 105 insertions(+), 5 deletions(-) diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 95ed9b49c06..73cc77c5985 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -13926,6 +13926,25 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }, ); + it("checks an automatic continuation that names no retry source against the task's previous run", async () => { + const source = await seedCommittedChatControlStop(); + const child = await seedChatAutomaticChild(source); + await db + .update(heartbeatRuns) + .set({ retryOfRunId: null, createdAt: new Date(Date.now() + 1_000) }) + .where(eq(heartbeatRuns.id, child.runId)); + expect( + (await readChatControlRecoveryStop(db, chatStopScope(source, child.runId))).kind, + ).toBe("stopped"); + await db + .update(chatPublications) + .set({ state: "pending" }) + .where(eq(chatPublications.id, source.publicationId)); + expect( + (await readChatControlRecoveryStop(db, chatStopScope(source, child.runId))).kind, + ).toBe("clear"); + }); + it("defers unresolved automatic ancestry at claim and records a distinct nonretrying failure after claim", async () => { const source = await seedCommittedChatControlStop(); await db diff --git a/server/src/__tests__/recovery-wake-herd.test.ts b/server/src/__tests__/recovery-wake-herd.test.ts index 5ef2ca9844e..2555d75ac59 100644 --- a/server/src/__tests__/recovery-wake-herd.test.ts +++ b/server/src/__tests__/recovery-wake-herd.test.ts @@ -836,4 +836,60 @@ suite("recovery wake herd", () => { await heartbeat.drainActiveRunExecutions(); } }, 120_000); + it("starts the assigned-work sweep's continuation of a task whose run was interrupted, ahead of fresher retries", async () => { + const agentId = await seedCompany(1); + const stranded = await task(agentId); + await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: stranded.id, invocationSource: "assignment", + status: "interrupted", errorCode: "server_shutdown_interrupted", livenessState: "needs_followup", + startedAt: new Date(Date.now() - 4 * 60 * 60_000), finishedAt: new Date(Date.now() - 3 * 60 * 60_000), + createdAt: new Date(Date.now() - 4 * 60 * 60_000), contextSnapshot: { issueId: stranded.id, taskId: stranded.id }, + resultJson: { stopReason: "interrupted", executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }); + await db.update(issues).set({ updatedAt: new Date(Date.now() - 3 * 60 * 60_000) }).where(eq(issues.id, stranded.id)); + const heartbeat = heartbeatService(db); + try { + const blocker = await queuedRecoveryRun(agentId, 1, 1); + await db.update(heartbeatRuns).set({ status: "running", startedAt: new Date() }).where(eq(heartbeatRuns.id, blocker.id)); + expect(await heartbeat.checkAssignedWorkInvariant({ companyId, issueId: stranded.id })) + .toMatchObject({ outcome: "repaired" }); + const [continuation] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.issueId, stranded.id), + eq(heartbeatRuns.status, "queued"))); + expect(continuation).toBeDefined(); + await db.update(heartbeatRuns).set({ createdAt: new Date(Date.now() - 2 * 60 * 60_000) }) + .where(eq(heartbeatRuns.id, continuation!.id)); + const fresher: Array = []; + for (let index = 0; index < 4; index += 1) fresher.push(await queuedRecoveryRun(agentId, 10 - index, 5)); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, blocker.id)); + await expect.poll(async () => { + await heartbeat.resumeQueuedRuns(); + return (await readRun(continuation!.id)).startedAt; + }, { timeout: 60_000, interval: 250 }).not.toBeNull(); + } finally { + await heartbeat.drainActiveRunExecutions(); + } + }, 120_000); + it("gives system recovery its share when the agent's recent starts were a person's recovery restarts", async () => { + const agentId = await seedCompany(1); + const system = await queuedRecoveryRun(agentId, 20, 5); + for (let index = 0; index < 3; index += 1) { + const issue = await task(agentId); + const [request] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "on_demand", triggerDetail: "manual", + reason: "issue_recovery_action_restored", status: "completed", requestedByActorType: "user", requestedByActorId: userId, + payload: { issueId: issue.id } }).returning(); + const startedAt = new Date(Date.now() - (10 - index) * 60_000); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: issue.id, invocationSource: "on_demand", + triggerDetail: "manual", status: "succeeded", wakeupRequestId: request!.id, startedAt, finishedAt: startedAt, + createdAt: new Date(startedAt.getTime() - 1_000), + contextSnapshot: { issueId: issue.id, taskId: issue.id, wakeReason: "issue_recovery_action_restored" } }).returning(); + await db.update(agentWakeupRequests).set({ runId: run!.id }).where(eq(agentWakeupRequests.id, request!.id)); + } + const people = [await queuedPersonRetry(agentId), await queuedPersonRetry(agentId)]; + const heartbeat = heartbeatService(db); + try { + await heartbeat.resumeQueuedRuns(); + expect((await readRun(system.id)).startedAt).not.toBeNull(); + for (const run of people) expect(await readRun(run.id)).toMatchObject({ status: "queued", startedAt: null }); + } finally { + await heartbeat.drainActiveRunExecutions(); + } + }, 120_000); }); diff --git a/server/src/services/chat-control-recovery-stop.ts b/server/src/services/chat-control-recovery-stop.ts index 900a752b260..34768a5378c 100644 --- a/server/src/services/chat-control-recovery-stop.ts +++ b/server/src/services/chat-control-recovery-stop.ts @@ -1,4 +1,4 @@ -import { and, eq, inArray, or, sql } from "drizzle-orm"; +import { and, desc, eq, inArray, lt, ne, or, sql } from "drizzle-orm"; import { runTaskContextColumn } from "./run-task-binding.js"; import { agentWakeupRequests, @@ -197,6 +197,7 @@ export async function readChatControlRecoveryStop( nativeIssueId: heartbeatRuns.nativeIssueId, wakeupRequestId: heartbeatRuns.wakeupRequestId, retryOfRunId: heartbeatRuns.retryOfRunId, + createdAt: heartbeatRuns.createdAt, contextSnapshot: runTaskContextColumn, }) .from(heartbeatRuns) @@ -287,8 +288,29 @@ export async function readChatControlRecoveryStop( wake.reason ?? "", )); if (!automatic) return { kind: "clear" }; - if (!id(run.retryOfRunId)) return { kind: "unresolved" }; - runId = run.retryOfRunId; + if (id(run.retryOfRunId)) { + runId = run.retryOfRunId; + continue; + } + const [previous] = await db + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, scope.companyId), + ne(heartbeatRuns.id, run.id), + lt(heartbeatRuns.createdAt, run.createdAt), + or( + eq(heartbeatRuns.issueId, scope.issueId), + eq(heartbeatRuns.nativeIssueId, scope.issueId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${scope.issueId}`, + ), + ), + ) + .orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)) + .limit(1); + if (!previous) return { kind: "clear" }; + runId = previous.id; continue; } const payload = record(action.payload); diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index e1ac0519ff0..cd8aee1e94f 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -2465,7 +2465,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende if (countsRecovery || (agedOldest && facts.length > 1)) { const oldestRecoveryAt = countsRecovery ? Math.min(...recoveryRuns.map(entry => entry.createdAt.getTime())) : Infinity; const since = new Date(Math.min(oldestRecoveryAt, agedOldest?.createdAt.getTime() ?? Infinity)); - const recentStarts = await db.select({ wakeReason: sql`${heartbeatRuns.contextSnapshot}->>'wakeReason'`, + const recentStarts = await db.select({ id: heartbeatRuns.id, wakeReason: sql`${heartbeatRuns.contextSnapshot}->>'wakeReason'`, startedAt: heartbeatRuns.startedAt, createdAt: heartbeatRuns.createdAt }) .from(heartbeatRuns) .where(and(eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id), @@ -2475,7 +2475,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende const startsSince = (at: number) => recentStarts.filter(row => row.startedAt && row.startedAt.getTime() >= at); if (countsRecovery) { const starts = startsSince(oldestRecoveryAt).slice(0, RECOVERY_DISPATCH_SHARE); - const lastRecovery = starts.findIndex(row => isRecoveryWakeReason(row.wakeReason)); + const recoveryStartIds = starts.flatMap(row => isRecoveryWakeReason(row.wakeReason) ? [row.id] : []); + const personStarts = new Set(recoveryStartIds.length + ? await listPersonOriginSourceRunIds(agent.companyId, recoveryStartIds) : []); + const lastRecovery = starts.findIndex(row => isRecoveryWakeReason(row.wakeReason) && !personStarts.has(row.id)); startsSinceRecovery = lastRecovery < 0 ? starts.length : lastRecovery; } if (agedOldest && facts.length > 1) {