From c5c587f16231b22dc60a3252a94e7fdef7a53437 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:14:47 -0600 Subject: [PATCH] fix(issues): start a deferred children-completed wake the release drain missed (#20180) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - A parent that delegates waits for its children, and is woken when they finish (`issue_children_completed`) > - Restart chaos t06 ended with both children done and the parent blocked forever > - The children finished while the parent's run was still going, so the wake was deferred as a follow-up. That run then ended with the agent setting the parent blocked, and the deferred wake was never started > - The durable re-drivers for deferred wakes cover only `issue_blockers_resolved`, so nothing else picks up a missed `issue_children_completed` > - This pull request makes both re-drivers cover children-completed wakes too > - The benefit is that a parent always hears that its children finished, even when the release drain misses the deferred wake ## Linked Issues or Issue Description Refs #19831 (durable ownership plan). Restart chaos wave `w-20261109T041248-s9501`, task t06 (claude, M1, K2), Daytona per-turn, master fa7950121d. **Problem:** parent CHA-10 (all times UTC): - 04:29:57: run 9cf49f52 started. - 04:32:35 and 04:32:43: its children CHA-13 and CHA-14 finished. - 04:32:43: `issue_children_completed` merged into the running turn. - 04:32:53: that wake was deferred as a follow-up (`issue_execution_deferred`, `deferred_issue_execution`, no run). - 04:33:13: the turn ended with the agent setting the parent `blocked`, a decision made before it saw the children's results. The deferred wake stayed deferred for the rest of the wave, including three later restarts. The parent sat blocked with no Retry. The deferred-receipt worker (`deferred-receipt-work.ts`) and the stranded-queue resume (`queue.ts`) select deferred dependency wakes with `issueBlockersResolvedWakePayloadSql`, which matches only `issue_blockers_resolved`. ## What Changed - `server/src/services/issue-dependency-wakeups.ts`: `dependencyResolvedWakePayloadSql` and `isDependencyResolvedWakePayload` replace the blockers-only predicates. They match both `issue_blockers_resolved` and `issue_children_completed` in `_paperclipWakeContext.wakeReason`. - The deferred-receipt worker and the stranded-queue resume use them, so a deferred children-completed wake is started once the parent has no live run, the same way a blockers-resolved one is. - No code comments added. ## Verification Red/green: - `deferred-execution-receipt-work.test.ts` "starts a deferred children-completed wake left behind when its parent's run blocked the parent (restart chaos t06)". It sets up a blocked parent whose last run succeeded, a done child, and the deferred `issue_children_completed` receipt as t06 left it. - On master: the receipt stays `deferred_issue_execution` (red). - With this PR: the receipt completes with exactly one `issue_children_completed` run. - `deferred-execution-receipt-work` and `heartbeat-dependency-scheduling` pass (60 tests). `tsc --noEmit -p server` is clean. ## Risks - The workers now also select deferred children-completed wakes. Admission is unchanged: they still pass the queue's own gates, and they start only when the issue has no live run, the same as blockers-resolved wakes. - No schema change. ## Model Used Anthropic Claude Opus 5.5 (`claude-opus-5-5`), via Claude Code, with tool use (shell, file editing). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above (#20069, #20108) - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes (no user docs affected) - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green (pending) - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups (pending) - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- .../deferred-execution-receipt-work.test.ts | 28 +++++++++++++++++++ .../heartbeat/deferred-receipt-work.ts | 4 +-- server/src/services/heartbeat/queue.ts | 8 +++--- .../src/services/issue-dependency-wakeups.ts | 13 ++++++--- 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/server/src/__tests__/deferred-execution-receipt-work.test.ts b/server/src/__tests__/deferred-execution-receipt-work.test.ts index 414ac9d5abf..76fe58ed5dd 100644 --- a/server/src/__tests__/deferred-execution-receipt-work.test.ts +++ b/server/src/__tests__/deferred-execution-receipt-work.test.ts @@ -185,6 +185,34 @@ describe("deferred receipt durable dispatch", () => { return { ...f, receipt, blocker }; } + it("starts a deferred children-completed wake left behind when its parent's run blocked the parent (restart chaos t06)", async () => { + const f = await fixture(); + await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: f.issue.id, status: "succeeded", + errorCode: null, contextSnapshot: { issueId: f.issue.id, wakeReason: "issue_assigned" } }).where(eq(heartbeatRuns.id, f.source.id)); + await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issue.id, runId: f.source.id, phase: "committed" }); + await db.update(issues).set({ status: "blocked", executionRunId: null, checkoutRunId: null }).where(eq(issues.id, f.issue.id)); + const [child] = await db.insert(issues).values({ companyId: f.companyId, title: "Part 1", parentId: f.issue.id, + status: "done", completedAt: new Date() }).returning(); + const [receipt] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, + source: "automation", reason: "issue_execution_deferred", status: "deferred_issue_execution", + requestedByActorType: "system", idempotencyKey: `issue_children_completed:${f.issue.id}`, + payload: { issueId: f.issue.id, childIssueIds: [child.id], _paperclipWakeContext: { issueId: f.issue.id, + taskId: f.issue.id, wakeReason: "issue_children_completed", source: "native_status_decision", childIssueIds: [child.id] } }, + }).returning(); + await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; + await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready; + await waitForReal(async () => { + await Promise.allSettled([...passes]); await heartbeat.drainActiveRunExecutions(); + const [saved] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.id)); + expect(saved.status, JSON.stringify({ errors })).toBe("completed"); + expect(saved.runId).not.toBeNull(); + }, { timeout: 15_000 }); + await finish(f.issue.id); + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId)); + expect(runs.filter(run => run.contextSnapshot?.wakeReason === "issue_children_completed")).toHaveLength(1); + await quiet(); + }); + it.each(["restart", "cleanup"] as const)("recovers a dependency wake after native audit %s into one continuation", async event => { const f = await strandedDependency(); const { receipt } = f; diff --git a/server/src/services/heartbeat/deferred-receipt-work.ts b/server/src/services/heartbeat/deferred-receipt-work.ts index 915d72b7b07..5a3e3bddd64 100644 --- a/server/src/services/heartbeat/deferred-receipt-work.ts +++ b/server/src/services/heartbeat/deferred-receipt-work.ts @@ -5,7 +5,7 @@ import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; import type { DeferredExecutionScope, DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; -import { issueBlockersResolvedWakePayloadSql } from "../issue-dependency-wakeups.js"; +import { dependencyResolvedWakePayloadSql } from "../issue-dependency-wakeups.js"; type Entry = { id: string; companyId: string; dueAt: number; immediate: boolean }; type NativeIntentOptions = DeferredExecutionScope & { @@ -75,7 +75,7 @@ export function registerDeferredReceiptWork( sql`jsonb_typeof(${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}') = 'array' and ${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`, sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`, - issueBlockersResolvedWakePayloadSql, + dependencyResolvedWakePayloadSql, sql`exists (select 1 from heartbeat_runs r where r.company_id = ${agentWakeupRequests.companyId} and r.runtime_mode = 'legacy' and r.status = 'cancelled' and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`, diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index dfeba70e0a1..be86fd31205 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -153,8 +153,8 @@ import { import { logActivity, publishActivity, type ActivityPublication } from "../activity-log.js"; import { ISSUE_BLOCKERS_RESOLVED_WAKE_REASON, - isIssueBlockersResolvedWakePayload, - issueBlockersResolvedWakePayloadSql, + isDependencyResolvedWakePayload, + dependencyResolvedWakePayloadSql, } from "../issue-dependency-wakeups.js"; import { resumeIfPauseAlreadyReleased } from "../issue-tree-resume.js"; import { @@ -1901,7 +1901,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende sql`jsonb_typeof(${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}') = 'array'`, sql`${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`, ), sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`, - issueBlockersResolvedWakePayloadSql), + dependencyResolvedWakePayloadSql), sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is null`, sql`NOT EXISTS (SELECT 1 FROM heartbeat_runs live_issue_run WHERE live_issue_run.company_id = ${issues.companyId} @@ -1912,7 +1912,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); const releasedNativeSources = new Set(); for (const { wake } of strandedQueues) { - const dependencyWake = isIssueBlockersResolvedWakePayload(wake.payload); + const dependencyWake = isDependencyResolvedWakePayload(wake.payload); if (!dependencyWake && !queuedCommentIdsFromWakePayload(wake.payload).length && !await readQueuedInteractionResponse(db, wake.companyId, String(wake.payload?.issueId), wake.payload)) continue; const [latest] = await db.select({ diff --git a/server/src/services/issue-dependency-wakeups.ts b/server/src/services/issue-dependency-wakeups.ts index 4d891d583a8..48591daaedc 100644 --- a/server/src/services/issue-dependency-wakeups.ts +++ b/server/src/services/issue-dependency-wakeups.ts @@ -5,13 +5,18 @@ import { agentWakeupRequests } from "@paperclipai/db"; export const ISSUE_BLOCKERS_RESOLVED_WAKE_REASON = "issue_blockers_resolved"; -export const issueBlockersResolvedWakePayloadSql = - sql`${agentWakeupRequests.payload}#>>'{_paperclipWakeContext,wakeReason}' = ${ISSUE_BLOCKERS_RESOLVED_WAKE_REASON}`; +export const ISSUE_CHILDREN_COMPLETED_WAKE_REASON = "issue_children_completed"; -export function isIssueBlockersResolvedWakePayload(payload: Record | null | undefined): boolean { +const DEPENDENCY_RESOLVED_WAKE_REASONS = [ISSUE_BLOCKERS_RESOLVED_WAKE_REASON, ISSUE_CHILDREN_COMPLETED_WAKE_REASON]; + +export const dependencyResolvedWakePayloadSql = + sql`${agentWakeupRequests.payload}#>>'{_paperclipWakeContext,wakeReason}' in (${sql.join( + DEPENDENCY_RESOLVED_WAKE_REASONS.map(reason => sql`${reason}`), sql`, `)})`; + +export function isDependencyResolvedWakePayload(payload: Record | null | undefined): boolean { const context = payload?._paperclipWakeContext; return context !== null && typeof context === "object" && !Array.isArray(context) - && (context as Record).wakeReason === ISSUE_BLOCKERS_RESOLVED_WAKE_REASON; + && DEPENDENCY_RESOLVED_WAKE_REASONS.includes(String((context as Record).wakeReason)); } /**