From 7e60ce3de18d58a2fa2772e7d28e690425837daf Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 22 Apr 2026 21:36:07 -0500 Subject: [PATCH] PAP-2079 create terminal recovery issues --- .../heartbeat-process-recovery.test.ts | 44 ++++++++++++++++++- server/src/services/heartbeat.ts | 44 +++---------------- server/src/services/recovery/service.ts | 31 ++++++++++++- 3 files changed, 78 insertions(+), 41 deletions(-) diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 5fd88dfb6be..c1e564ddf5b 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -800,11 +800,28 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { it("blocks the issue when process-loss retry is exhausted and the immediate continuation recovery also fails", async () => { mockAdapterExecute.mockRejectedValueOnce(new Error("continuation recovery failed")); - const { agentId, runId, issueId } = await seedRunFixture({ + const { companyId, agentId, runId, issueId } = await seedRunFixture({ agentStatus: "idle", processPid: 999_999_999, processLossRetryCount: 1, }); + const resolvedBlockerId = randomUUID(); + const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`; + await db.insert(issues).values({ + id: resolvedBlockerId, + companyId, + title: "Already completed prerequisite", + status: "done", + priority: "medium", + issueNumber: 2, + identifier: `${issuePrefix}-2`, + }); + await db.insert(issueRelations).values({ + companyId, + issueId: resolvedBlockerId, + relatedIssueId: issueId, + type: "blocks", + }); const heartbeat = heartbeatService(db); const result = await heartbeat.reapOrphanedRuns(); @@ -831,7 +848,29 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { ); expect(blockedIssue?.status).toBe("blocked"); expect(blockedIssue?.executionRunId).toBeNull(); - expect(blockedIssue?.checkoutRunId).toBe(continuationRun?.id ?? null); + expect(blockedIssue?.checkoutRunId).toBeNull(); + if (!continuationRun?.id) throw new Error("Expected continuation recovery run to exist"); + + const recovery = await expectStrandedRecoveryArtifacts({ + companyId, + agentId, + issueId, + runId: continuationRun.id, + previousStatus: "in_progress", + retryReason: "issue_continuation_needed", + }); + + const blockerRelations = await db + .select() + .from(issueRelations) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.relatedIssueId, issueId), + eq(issueRelations.type, "blocks"), + ), + ); + expect(blockerRelations.map((relation) => relation.issueId)).toEqual([recovery.id]); const comments = await waitForValue(async () => { const rows = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId)); @@ -839,6 +878,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }); expect(comments).toHaveLength(1); expect(comments[0]?.body).toContain("retried continuation"); + expect(comments[0]?.body).toContain(`Recovery issue: [${recovery.identifier}]`); }); it("schedules a bounded retry for codex transient upstream failures instead of blocking the issue immediately", async () => { diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index d170729e550..e14906ae7ff 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -5519,15 +5519,7 @@ export function heartbeatService(db: Db) { } let issue = await tx - .select({ - id: issues.id, - companyId: issues.companyId, - identifier: issues.identifier, - status: issues.status, - assigneeAgentId: issues.assigneeAgentId, - assigneeUserId: issues.assigneeUserId, - executionRunId: issues.executionRunId, - }) + .select() .from(issues) .where( and( @@ -5776,17 +5768,9 @@ export function heartbeatService(db: Db) { status: issue.status as "todo" | "in_progress", latestRun: run, }); - await tx - .update(issues) - .set({ - status: "blocked", - updatedAt: new Date(), - }) - .where(eq(issues.id, issue.id)); return { kind: "blocked" as const, - issueId: issue.id, - issueIdentifier: issue.identifier, + issue, previousStatus: issue.status, comment, }; @@ -5866,25 +5850,11 @@ export function heartbeatService(db: Db) { }); if (promotionResult?.kind === "blocked") { - await issuesSvc.addComment(promotionResult.issueId, promotionResult.comment, {}); - await logActivity(db, { - companyId: run.companyId, - actorType: "system", - actorId: "system", - agentId: null, - runId: run.id, - action: "issue.updated", - entityType: "issue", - entityId: promotionResult.issueId, - details: { - identifier: promotionResult.issueIdentifier, - status: "blocked", - previousStatus: promotionResult.previousStatus, - source: "heartbeat.release_issue_execution_and_promote", - latestRunId: run.id, - latestRunStatus: run.status, - latestRunErrorCode: run.errorCode ?? null, - }, + await recovery.escalateStrandedAssignedIssue({ + issue: promotionResult.issue, + previousStatus: promotionResult.previousStatus as "todo" | "in_progress", + latestRun: run, + comment: promotionResult.comment, }); return; } diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 13c0eb79e9a..5e9537bb7bc 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -1304,6 +1304,28 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) .then((rows) => rows.map((row) => row.blockerIssueId)); } + async function existingUnresolvedBlockerIssueIds(companyId: string, issueId: string) { + return db + .select({ blockerIssueId: issueRelations.issueId }) + .from(issueRelations) + .innerJoin( + issues, + and( + eq(issues.companyId, issueRelations.companyId), + eq(issues.id, issueRelations.issueId), + ), + ) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.relatedIssueId, issueId), + eq(issueRelations.type, "blocks"), + notInArray(issues.status, ["done", "cancelled"]), + ), + ) + .then((rows) => rows.map((row) => row.blockerIssueId)); + } + async function escalateStrandedAssignedIssue(input: { issue: typeof issues.$inferSelect; previousStatus: "todo" | "in_progress"; @@ -1315,7 +1337,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) previousStatus: input.previousStatus, latestRun: input.latestRun, }); - const blockerIds = await existingBlockerIssueIds(input.issue.companyId, input.issue.id); + const blockerIds = await existingUnresolvedBlockerIssueIds(input.issue.companyId, input.issue.id); const nextBlockerIds = recoveryIssue ? [...new Set([...blockerIds, recoveryIssue.id])] : blockerIds; @@ -1332,7 +1354,11 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) `- Recovery issue: ${issueUiLink({ identifier: recoveryIssue.identifier, id: recoveryIssue.id }, prefix)}`, "- Next action: the recovery owner should either restore a live execution path or record the manual resolution, then mark the recovery issue done.", ].join("\n") - : ""; + : [ + "", + "- Recovery issue: none created because Paperclip could not find an invokable manager, creator, or executive owner with budget available.", + "- Next action: a board operator should assign an invokable recovery owner, fix the agent/runtime state, or record an intentional manual resolution.", + ].join("\n"); await issuesSvc.addComment(input.issue.id, `${input.comment}${recoveryLine}`, {}); @@ -2085,6 +2111,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) return { buildRunOutputSilence, + escalateStrandedAssignedIssue, recordWatchdogDecision, scanSilentActiveRuns, reconcileStrandedAssignedIssues,