diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index d66c2617ea4..242d6fff73a 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -2180,7 +2180,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { expect(wakeups).toHaveLength(1); }); - it("records productive continuation instead of recovery when the latest automatic continuation succeeded", async () => { + it("re-enqueues recovery when the latest automatic continuation made progress but left no live path", async () => { const { agentId, issueId, runId } = await seedStrandedIssueFixture({ status: "in_progress", runStatus: "succeeded", @@ -2190,11 +2190,11 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const heartbeat = heartbeatService(db); const result = await heartbeat.reconcileStrandedAssignedIssues(); - expect(result.continuationRequeued).toBe(0); - expect(result.productiveContinuationObserved).toBe(1); + expect(result.continuationRequeued).toBe(1); + expect(result.productiveContinuationObserved).toBe(0); expect(result.successfulContinuationObserved).toBe(0); expect(result.escalated).toBe(0); - expect(result.issueIds).toEqual([]); + expect(result.issueIds).toEqual([issueId]); const issue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null); expect(issue?.status).toBe("in_progress"); @@ -2206,10 +2206,68 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.agentId, agentId)); - expect(runs.map((row) => row.id)).toEqual([runId]); + expect(runs).toHaveLength(2); + const retryRun = runs.find((row) => row.id !== runId); + expect(retryRun?.contextSnapshot as Record | undefined).toMatchObject({ + issueId, + taskId: issueId, + retryReason: "issue_continuation_needed", + retryOfRunId: runId, + source: "issue.productive_terminal_continuation_recovery", + }); const wakeups = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, agentId)); - expect(wakeups).toHaveLength(1); + expect(wakeups).toHaveLength(2); + }); + + it("does not treat a productive terminal run as healthy when in-progress work has no live path", async () => { + const { companyId, agentId, issueId } = await seedStrandedIssueFixture({ + status: "in_progress", + runStatus: "succeeded", + retryReason: "issue_continuation_needed", + livenessState: "advanced", + }); + const heartbeat = heartbeatService(db); + + const sourceIssue = await db.select().from(issues).where(eq(issues.id, issueId)).then((rows) => rows[0] ?? null); + expect(sourceIssue).toMatchObject({ + status: "in_progress", + assigneeAgentId: agentId, + assigneeUserId: null, + executionRunId: null, + }); + + const activeRuns = await db + .select() + .from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.status, ["queued", "running"]))); + expect(activeRuns).toHaveLength(0); + + const liveWakeups = await db + .select() + .from(agentWakeupRequests) + .where(and(eq(agentWakeupRequests.companyId, companyId), inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]))); + expect(liveWakeups).toHaveLength(0); + + const result = await heartbeat.reconcileStrandedAssignedIssues(); + expect(result.productiveContinuationObserved).toBe(0); + expect(result.continuationRequeued + result.escalated).toBe(1); + expect(result.issueIds).toEqual([issueId]); + + const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId)); + const recoveryIssues = await db + .select() + .from(issues) + .where(and(eq(issues.companyId, companyId), eq(issues.originKind, "stranded_issue_recovery"))); + const followupWakeups = await db + .select() + .from(agentWakeupRequests) + .where(and(eq(agentWakeupRequests.companyId, companyId), inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]))); + const followupRuns = await db + .select() + .from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.status, ["queued", "running", "scheduled_retry"]))); + expect(comments.length + recoveryIssues.length + followupWakeups.length + followupRuns.length).toBeGreaterThan(0); }); it("does not reconcile user-assigned work through the agent stranded-work recovery path", async () => { diff --git a/server/src/__tests__/run-continuations.test.ts b/server/src/__tests__/run-continuations.test.ts index a9e1e7b0006..82b132351d7 100644 --- a/server/src/__tests__/run-continuations.test.ts +++ b/server/src/__tests__/run-continuations.test.ts @@ -106,6 +106,24 @@ describe("run liveness continuations", () => { expect(decision.nextAttempt).toBe(2); }); + it("leaves advanced terminal runs to stranded issue recovery instead of bounded liveness continuation", () => { + const decision = decideRunLivenessContinuation({ + run: run(), + issue: issue(), + agent: agent(), + livenessState: "advanced", + livenessReason: "Run produced concrete action evidence: created an issue comment", + nextAction: "Resume the implementation from the remaining acceptance criteria.", + budgetBlocked: false, + idempotentWakeExists: false, + }); + + expect(decision).toEqual({ + kind: "skip", + reason: "liveness state is not actionable for continuation", + }); + }); + it("does not enqueue a third continuation and returns an exhaustion comment", () => { const decision = decideRunLivenessContinuation({ run: run({ continuationAttempt: 2 }), diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index ec7213515e7..368f34ef14b 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -345,7 +345,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) .where( and( eq(agentWakeupRequests.companyId, companyId), - eq(agentWakeupRequests.status, "deferred_issue_execution"), + eq(agentWakeupRequests.status, "deferred_issue_execution"), sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`, ), ) @@ -1706,12 +1706,37 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) continue; } if (isSuccessfulInProgressContinuationRun(latestRun)) { - if (isProductiveContinuationRun(latestRun)) { - result.productiveContinuationObserved += 1; + const successfulRun = latestRun; + if (!successfulRun) { + result.skipped += 1; + continue; + } + + if (!isProductiveContinuationRun(successfulRun)) { + result.successfulContinuationObserved += 1; + result.skipped += 1; + continue; + } + + if (await isInvocationBudgetBlocked(issue, agentId)) { + result.skipped += 1; + continue; + } + + const queued = await enqueueStrandedIssueRecovery({ + issueId: issue.id, + agentId, + reason: "issue_continuation_needed", + retryReason: "issue_continuation_needed", + source: "issue.productive_terminal_continuation_recovery", + retryOfRunId: successfulRun.id, + }); + if (queued) { + result.continuationRequeued += 1; + result.issueIds.push(issue.id); } else { - result.successfulContinuationObserved += 1; + result.skipped += 1; } - result.skipped += 1; continue; } if (didAutomaticRecoveryFail(latestRun, "issue_continuation_needed")) {