diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 73cc77c5985..0a662626ef2 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -17312,6 +17312,40 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { } }); + it.each(["queued", "claimed"] as const)( + "does not queue a second continuation while the finished run's own status wake (%s) awaits dispatch (Gate C C1 t169)", + async (intentStatus) => { + const { companyId, agentId, issueId, runId } = + await seedStrandedIssueFixture({ + status: "in_progress", + runStatus: "succeeded", + livenessState: "advanced", + }); + await db.insert(agentWakeupRequests).values({ + companyId, + agentId, + source: "automation", + triggerDetail: "system", + reason: "issue_continuation_needed", + status: intentStatus, + claimedAt: intentStatus === "claimed" ? new Date() : null, + requestedByActorType: "system", + requestedByActorId: "native-status-committer", + idempotencyKey: `agent_tool_call_loop:${runId}`, + payload: { issueId, taskId: issueId, sourceRunId: runId, continuationCause: "agent_tool_call_loop" }, + }); + await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issueId }).where(eq(heartbeatRuns.id, runId)); + const heartbeat = heartbeatService(db); + + const result = await heartbeat.reconcileStrandedAssignedIssues(); + expect(result.continuationRequeued).toBe(0); + expect(result.escalated).toBe(0); + expect(result.skipped).toBe(1); + expect(await selectRecoveryRunsByAgent(agentId)).toHaveLength(1); + expect((await selectRecoveryIssueById(issueId))?.status).toBe("in_progress"); + }, + ); + it("does not replenish exhausted legacy repair from a recent comment", async () => { const { companyId, agentId, issueId, runId } = await seedStrandedIssueFixture({ diff --git a/server/src/__tests__/task-interruption-query.test.ts b/server/src/__tests__/task-interruption-query.test.ts index 0b81445c7a2..cf28f663f43 100644 --- a/server/src/__tests__/task-interruption-query.test.ts +++ b/server/src/__tests__/task-interruption-query.test.ts @@ -747,6 +747,43 @@ describe.skipIf(!support.supported)("task interruption latest run query", () => taskId: reassigned!.id, agentAdapter: "process", })); }); + describe("a newer continuation dropped before it started (Gate C C1 t169)", () => { + const seed = async (newer: Partial) => { + const companyId = randomUUID(), agentId = randomUUID(), runningId = randomUUID(), droppedId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Continuing task", issuePrefix: `C${companyId.slice(0, 7)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Continuing agent", adapterType: "paperclip_runner" }); + const [issue] = await db.insert(issues).values({ companyId, title: "Keep working", + status: "in_progress", assigneeAgentId: agentId }).returning(); + const now = Date.now(); + await db.insert(heartbeatRuns).values({ id: runningId, companyId, agentId, nativeIssueId: issue!.id, scopeKind: "issue", + status: "running", runtimeMode: "native", createdAt: new Date(now - 60_000), startedAt: new Date(now - 59_000) }); + await db.insert(heartbeatRuns).values({ id: droppedId, companyId, agentId, nativeIssueId: issue!.id, scopeKind: "issue", + status: "cancelled", runtimeMode: "native", createdAt: new Date(now - 30_000), finishedAt: new Date(now - 30_000), + contextSnapshot: { issueId: issue!.id, wakeReason: "issue_continuation_needed" }, ...newer }); + return { companyId, issue: issue!, runningId, droppedId }; + }; + + it.each(["recovery_no_longer_needed", "issue_closed_before_start"])( + "shows the running run, not a failed attempt (%s)", async errorCode => { + const f = await seed({ errorCode, error: "The task no longer needed this recovery run when its turn came." }); + for (const presentation of [(await taskInterruptionPresentations(db, f.companyId, [f.issue])).get(f.issue.id), + await taskInterruptionPresentation(db, f.issue)]) { + expect(presentation).toMatchObject({ state: "working", canRetry: false, observedRunId: f.runningId }); + } + }); + + it("still reports a cancelled run that had started", async () => { + const f = await seed({ errorCode: "recovery_no_longer_needed", startedAt: new Date(Date.now() - 29_000) }); + expect((await taskInterruptionPresentations(db, f.companyId, [f.issue])).get(f.issue.id)) + .toMatchObject({ state: "failed", canRetry: true, observedRunId: f.droppedId }); + }); + + it("still reports a run a person cancelled before it started", async () => { + const f = await seed({ errorCode: "cancelled", resultJson: { cancelledByActorType: "user", cancelledByUserId: "board" } }); + expect((await taskInterruptionPresentations(db, f.companyId, [f.issue])).get(f.issue.id)) + .toMatchObject({ state: "failed", canRetry: true, observedRunId: f.droppedId }); + }); + }); // N-03: ONB-1 on staging waited 16 minutes on an overdue workspace_busy // retry. Nothing failed, yet the banner offered a Retry that cannot help. it.each(["workspace_busy", "ai_connection_busy", "ai_connection_pool_wait", "runner_revision_wait", "storage_unavailable_wait"])( diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 0cb4576f41e..75efec067d3 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -42,6 +42,7 @@ import { import { CONTROLLER_LEASE_LOST_FAILURE_CODE, hasLiveLegacyController, legacyControllerBootId, liveLegacyControllerExpiresAt } from "../legacy-controller-lease.js"; import { isCompletedOnboardingHandoffWake } from "../chat-completion-delivery.js"; import { instanceSettingsService } from "../instance-settings.js"; +import { NATIVE_STATUS_COMMITTER_ACTOR_ID } from "../issue-rewake-throttle.js"; import { isWaitingConversation, settleConversationTurnForRun, deliverConversationComments } from "../agent-conversations.js"; import { and, @@ -1259,7 +1260,15 @@ export function recoveryService( .where( and( eq(agentWakeupRequests.companyId, companyId), - eq(agentWakeupRequests.status, "deferred_issue_execution"), + or( + eq(agentWakeupRequests.status, "deferred_issue_execution"), + and( + eq(agentWakeupRequests.requestedByActorType, "system"), + eq(agentWakeupRequests.requestedByActorId, NATIVE_STATUS_COMMITTER_ACTOR_ID), + inArray(agentWakeupRequests.status, ["queued", "claimed"]), + isNull(agentWakeupRequests.runId), + ), + ), sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`, agentId ? eq(agentWakeupRequests.agentId, agentId) : sql`true`, ), diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index ca13eebd909..87b18917f83 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -440,8 +440,10 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i const requestedUuid = sql`case when ${requestedIssueId} ~ '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' then ${requestedIssueId}::uuid else null end`; const legacyIssue = sql`${heartbeatRuns.contextSnapshot}->>'issueId'`; + const attempt = sql`not (${heartbeatRuns.status} = 'cancelled' and ${heartbeatRuns.startedAt} is null + and coalesce(${heartbeatRuns.errorCode}, '') in ('recovery_no_longer_needed', 'issue_closed_before_start'))`; const candidate = (condition: ReturnType) => db.select({ id: heartbeatRuns.id, createdAt: heartbeatRuns.createdAt }) - .from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), condition)) + .from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), condition, attempt)) .orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); const candidates = unionAll( candidate(eq(heartbeatRuns.nativeIssueId, requestedUuid)),