From b1ef1f3022620ce231bc09cbfcc4c777ad2a194b Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:46:21 -0600 Subject: [PATCH] fix(recovery): settle a run's late exhaustion hold into its recorded fresh continuation (#21217) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - When a run cannot be resumed, #21135 continues the task in one fresh run from its saved work, with no person action > - In Runner Full-Stack E2E 43321773136, two tasks whose run lost its Runner three times recorded that continuation ("Unfinished run continues in a fresh run from saved work", `delivery: recorded`), but no continuation run started in the next 5 minutes > - The native executor saves its exhaustion hold (`native_session_retry_exhausted`, board-owned) in the same failure path. When that hold lands after the decision, the decision is idempotent per run and returns `already_continued`, so nothing settles the new hold. The hold then blocks the continuation's delivery. > - This pull request settles such later reconciliation holds of the same run into the recorded continuation > - The benefit is that the recorded continuation starts, and the task does not wait on a person for an infrastructure fault ## Linked Issues or Issue Description Refs #21135 **What happened?** Runner Full-Stack E2E 43321773136, cases `recovery.explain-stall` and `recovery.source-action` (OpenCode, native Runner): the server logged the fresh-continuation decision at 22:51:30 and 22:51:55. Then the same run logged "native result could not be marked workspace_failed" and "heartbeat execution failed" (`native_session_retry_exhausted`). No continuation run was created in the following 5 minutes. The ownership pass kept reporting one `execution_reconciliation_hold` in progress. **Expected behavior** One fresh continuation starts from saved work after an infrastructure loss, whatever order the failure path saves its records in. **Steps to reproduce** 1. Record a fresh continuation for a failed native run. 2. Save an active `native_session_retry_exhausted` hold for the same run afterwards. 3. Run the disposition and continuation delivery workers. Before this change, no successor run is created. ## What Changed - `server/src/services/fresh-continuation-fallback.ts`: when a continuation is already recorded for the run, the decision settles the run's other owned reconciliation holds (blocking, not a human gate) into it: resolved, `restored`, system-owned, the person-facing text, `coalescedIntoRecoveryActionId` and the primary's delivery state. A task those holds blocked returns to `todo`. The continuation delivery worker is notified. - `server/src/services/fresh-continuation-fallback.test.ts`: regression test for a late exhaustion hold. ## Verification - RED before the change: `npx vitest run src/services/fresh-continuation-fallback.test.ts -t "exhaustion hold lands after"` failed with `expected [] to have a length of 1 but got +0`. - GREEN after: the full file passes (52/52). `issue-recovery-actions.test.ts` and `execution-control-work.test.ts` also pass (191 tests in the three files). Server `tsc --noEmit` is clean. - The E2E recovery cases in #21210 check the continuation starts and finishes on the next cohort. ## Risks - Low. Only holds of the same run that the fallback already owns (reconciliation causes, blocking, no person gate) are settled, and only after a continuation exists for that run. Human gates and workspace restore holds are untouched. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, used through Claude Code with tool use and code execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../fresh-continuation-fallback.test.ts | 29 +++++++++++++++ .../services/fresh-continuation-fallback.ts | 35 ++++++++++++++++++- 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/server/src/services/fresh-continuation-fallback.test.ts b/server/src/services/fresh-continuation-fallback.test.ts index 21a67499d96..8cf3d65df14 100644 --- a/server/src/services/fresh-continuation-fallback.test.ts +++ b/server/src/services/fresh-continuation-fallback.test.ts @@ -166,6 +166,35 @@ suite("fresh-continuation fallback", () => { // Staging: the person had already pressed Retry, and the automatic successor // could not carry that authorization forward. That is the system's failure, // not the person's: the task continues under its responsible user. + it("starts the continuation when the failed run's exhaustion hold lands after the continuation was recorded", async () => { + const f = await unfinishedRun({ runtime: "native", errorCode: "native_runner_process_exited", + error: "native_session_recovery_failed: Error: runner_local_connect_failed: runnerd exited unexpectedly" }); + await settleUnrecoverableExecutions(db); + await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: f.issueId, kind: "active_run_watchdog", status: "active", + ownerType: "board", returnOwnerAgentId: agentId, cause: "native_session_retry_exhausted", fingerprint: `native-retry:${f.runId}`, + nextAction: "Automatic recovery stopped.", maxAttempts: 3, + evidence: { runId: f.runId, coordinatorAttempt: 3, recoveryDisposition: "native_session_retry_exhausted" } }); + await deliver(); + await continueHeldExecutions(db); await settleUnrecoverableExecutions(db); await deliver(); + expect(await successors(f.runId)).toHaveLength(1); + }, 60_000); + + it("keeps a person's newer block when the failed run's exhaustion hold lands after the continuation was recorded", async () => { + const f = await unfinishedRun({ runtime: "native", errorCode: "native_runner_process_exited", + error: "native_session_recovery_failed: Error: runner_local_connect_failed: runnerd exited unexpectedly" }); + await settleUnrecoverableExecutions(db); + const decisionId = randomUUID(); + await db.update(issues).set({ status: "blocked", statusVersion: 9, lastStatusDecisionId: decisionId, blockedTransitionAt: new Date() }) + .where(eq(issues.id, f.issueId)); + const [late] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: f.issueId, kind: "active_run_watchdog", status: "active", + ownerType: "board", returnOwnerAgentId: agentId, cause: "native_session_retry_exhausted", fingerprint: `native-retry:${f.runId}`, + nextAction: "Automatic recovery stopped.", maxAttempts: 3, + evidence: { runId: f.runId, coordinatorAttempt: 3, recoveryDisposition: "native_session_retry_exhausted", nativeFailureBlock: { runId: f.runId, statusVersion: 4 } } }).returning(); + await settleUnrecoverableExecutions(db); + expect(await issue(f.issueId)).toMatchObject({ status: "blocked", lastStatusDecisionId: decisionId }); + expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, late!.id)))[0]).toMatchObject({ status: "resolved", ownerType: "system" }); + }, 60_000); + it("continues a task whose automatic retry lost the person's continuation authorization", async () => { expect(PERSON_OWNED_FAILURE_CODES).not.toContain("continuation_user_authorization_missing"); const f = await unfinishedRun({ runtime: "legacy", errorCode: "continuation_user_authorization_missing", diff --git a/server/src/services/fresh-continuation-fallback.ts b/server/src/services/fresh-continuation-fallback.ts index 092084af912..27b9b3709c9 100644 --- a/server/src/services/fresh-continuation-fallback.ts +++ b/server/src/services/fresh-continuation-fallback.ts @@ -339,7 +339,10 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< or(sql`${issueRecoveryActions.evidence}->>'runId' = ${run.id}`, sql`${issueRecoveryActions.evidence}->>'sourceRunId' = ${run.id}`))) .orderBy(asc(issueRecoveryActions.createdAt), asc(issueRecoveryActions.id)).for("update"); const prior = holds.find(hold => object(hold.evidence.freshContinuation).sourceRunId === run.id); - if (prior) return { kind: "already_continued", recoveryActionId: prior.id }; + if (prior) { + await settleLateHolds(tx, { companyId, issue, run, prior, holds: holds.filter(isOwnedReconciliationHold), now }); + return { kind: "already_continued", recoveryActionId: prior.id }; + } // The holds this decision owns: reconciliation holds that still block the // task. Workspace restore/repair holds stay with workspace recovery, and // resolved history is never rewritten. @@ -593,6 +596,36 @@ export async function continueFreshInTransaction(tx: Db, input: Input): Promise< return { kind: "continued", recoveryActionId: primaryId, delivery, isolation }; } +async function settleLateHolds(tx: Db, input: { companyId: string; issue: typeof issues.$inferSelect; run: typeof heartbeatRuns.$inferSelect; + prior: typeof issueRecoveryActions.$inferSelect; holds: Array; now: Date }) { + const { companyId, issue, run, prior, holds, now } = input; + if (!holds.length) return; + await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionContinuation); + await notifyStatusCardIssueChange(tx, companyId); + await notifyRoutineExecutionStatus(tx, issue.id, companyId); + const ownsBlock = issue.status === "blocked" && holds.some(hold => holdOwnsBlockedStatus(hold, issue, run)); + const delivery = String(prior.evidence.continuationDelivery); + const followed = ["delivered", "invalidated"].includes(delivery) ? delivery : "coalesced"; + for (const hold of holds) { + const { automaticRecovery, [FRESH_CONTINUATION_CHECK_KEY]: _check, ...evidence } = hold.evidence; + await tx.update(issueRecoveryActions).set({ status: "resolved", outcome: "restored", resolvedAt: now, updatedAt: now, + ownerType: "system", ownerAgentId: null, ownerUserId: null, wakePolicy: null, monitorPolicy: null, nextAction: USER_TEXT, + resolutionNote: "continued_fresh_from_last_saved", + evidence: { ...evidence, ...(automaticRecovery !== undefined ? { automaticRecoveryHistory: automaticRecovery } : {}), + continuationDelivery: followed, coalescedIntoRecoveryActionId: prior.id, + ...(typeof prior.evidence.continuationRunId === "string" ? { continuationRunId: prior.evidence.continuationRunId } : {}) }, + }).where(and(eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.id, hold.id))); + } + if (ownsBlock) { + await tx.update(issues).set({ status: "todo", unblockDescriptor: null, blockedTransitionAt: null, + blockedOwnerNotifiedAt: null, updatedAt: now }).where(and(eq(issues.companyId, companyId), eq(issues.id, issue.id))); + await logActivity(tx, { companyId, actorType: "system", actorId: FRESH_CONTINUATION_ACTOR, action: "issue.updated", + entityType: "issue", entityId: issue.id, agentId: run.agentId, runId: run.id, + details: { status: "todo", previousStatus: "blocked", reason: "fresh_continuation", recoveryActionId: prior.id, + _previous: { status: "blocked" } } }); + } +} + /** Records why a hold was not continued yet, and when to look again. */ export async function recordFreshContinuationCheck(db: Db, input: { companyId: string; actionId: string }, result: FreshContinuationResult, now = new Date()) {