diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 95ed9b49c06..73cc77c5985 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -13926,6 +13926,25 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }, ); + it("checks an automatic continuation that names no retry source against the task's previous run", async () => { + const source = await seedCommittedChatControlStop(); + const child = await seedChatAutomaticChild(source); + await db + .update(heartbeatRuns) + .set({ retryOfRunId: null, createdAt: new Date(Date.now() + 1_000) }) + .where(eq(heartbeatRuns.id, child.runId)); + expect( + (await readChatControlRecoveryStop(db, chatStopScope(source, child.runId))).kind, + ).toBe("stopped"); + await db + .update(chatPublications) + .set({ state: "pending" }) + .where(eq(chatPublications.id, source.publicationId)); + expect( + (await readChatControlRecoveryStop(db, chatStopScope(source, child.runId))).kind, + ).toBe("clear"); + }); + it("defers unresolved automatic ancestry at claim and records a distinct nonretrying failure after claim", async () => { const source = await seedCommittedChatControlStop(); await db diff --git a/server/src/__tests__/recovery-wake-herd.test.ts b/server/src/__tests__/recovery-wake-herd.test.ts index 5ef2ca9844e..2555d75ac59 100644 --- a/server/src/__tests__/recovery-wake-herd.test.ts +++ b/server/src/__tests__/recovery-wake-herd.test.ts @@ -836,4 +836,60 @@ suite("recovery wake herd", () => { await heartbeat.drainActiveRunExecutions(); } }, 120_000); + it("starts the assigned-work sweep's continuation of a task whose run was interrupted, ahead of fresher retries", async () => { + const agentId = await seedCompany(1); + const stranded = await task(agentId); + await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: stranded.id, invocationSource: "assignment", + status: "interrupted", errorCode: "server_shutdown_interrupted", livenessState: "needs_followup", + startedAt: new Date(Date.now() - 4 * 60 * 60_000), finishedAt: new Date(Date.now() - 3 * 60 * 60_000), + createdAt: new Date(Date.now() - 4 * 60 * 60_000), contextSnapshot: { issueId: stranded.id, taskId: stranded.id }, + resultJson: { stopReason: "interrupted", executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }); + await db.update(issues).set({ updatedAt: new Date(Date.now() - 3 * 60 * 60_000) }).where(eq(issues.id, stranded.id)); + const heartbeat = heartbeatService(db); + try { + const blocker = await queuedRecoveryRun(agentId, 1, 1); + await db.update(heartbeatRuns).set({ status: "running", startedAt: new Date() }).where(eq(heartbeatRuns.id, blocker.id)); + expect(await heartbeat.checkAssignedWorkInvariant({ companyId, issueId: stranded.id })) + .toMatchObject({ outcome: "repaired" }); + const [continuation] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.issueId, stranded.id), + eq(heartbeatRuns.status, "queued"))); + expect(continuation).toBeDefined(); + await db.update(heartbeatRuns).set({ createdAt: new Date(Date.now() - 2 * 60 * 60_000) }) + .where(eq(heartbeatRuns.id, continuation!.id)); + const fresher: Array = []; + for (let index = 0; index < 4; index += 1) fresher.push(await queuedRecoveryRun(agentId, 10 - index, 5)); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, blocker.id)); + await expect.poll(async () => { + await heartbeat.resumeQueuedRuns(); + return (await readRun(continuation!.id)).startedAt; + }, { timeout: 60_000, interval: 250 }).not.toBeNull(); + } finally { + await heartbeat.drainActiveRunExecutions(); + } + }, 120_000); + it("gives system recovery its share when the agent's recent starts were a person's recovery restarts", async () => { + const agentId = await seedCompany(1); + const system = await queuedRecoveryRun(agentId, 20, 5); + for (let index = 0; index < 3; index += 1) { + const issue = await task(agentId); + const [request] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "on_demand", triggerDetail: "manual", + reason: "issue_recovery_action_restored", status: "completed", requestedByActorType: "user", requestedByActorId: userId, + payload: { issueId: issue.id } }).returning(); + const startedAt = new Date(Date.now() - (10 - index) * 60_000); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: issue.id, invocationSource: "on_demand", + triggerDetail: "manual", status: "succeeded", wakeupRequestId: request!.id, startedAt, finishedAt: startedAt, + createdAt: new Date(startedAt.getTime() - 1_000), + contextSnapshot: { issueId: issue.id, taskId: issue.id, wakeReason: "issue_recovery_action_restored" } }).returning(); + await db.update(agentWakeupRequests).set({ runId: run!.id }).where(eq(agentWakeupRequests.id, request!.id)); + } + const people = [await queuedPersonRetry(agentId), await queuedPersonRetry(agentId)]; + const heartbeat = heartbeatService(db); + try { + await heartbeat.resumeQueuedRuns(); + expect((await readRun(system.id)).startedAt).not.toBeNull(); + for (const run of people) expect(await readRun(run.id)).toMatchObject({ status: "queued", startedAt: null }); + } finally { + await heartbeat.drainActiveRunExecutions(); + } + }, 120_000); }); diff --git a/server/src/services/chat-control-recovery-stop.ts b/server/src/services/chat-control-recovery-stop.ts index 900a752b260..34768a5378c 100644 --- a/server/src/services/chat-control-recovery-stop.ts +++ b/server/src/services/chat-control-recovery-stop.ts @@ -1,4 +1,4 @@ -import { and, eq, inArray, or, sql } from "drizzle-orm"; +import { and, desc, eq, inArray, lt, ne, or, sql } from "drizzle-orm"; import { runTaskContextColumn } from "./run-task-binding.js"; import { agentWakeupRequests, @@ -197,6 +197,7 @@ export async function readChatControlRecoveryStop( nativeIssueId: heartbeatRuns.nativeIssueId, wakeupRequestId: heartbeatRuns.wakeupRequestId, retryOfRunId: heartbeatRuns.retryOfRunId, + createdAt: heartbeatRuns.createdAt, contextSnapshot: runTaskContextColumn, }) .from(heartbeatRuns) @@ -287,8 +288,29 @@ export async function readChatControlRecoveryStop( wake.reason ?? "", )); if (!automatic) return { kind: "clear" }; - if (!id(run.retryOfRunId)) return { kind: "unresolved" }; - runId = run.retryOfRunId; + if (id(run.retryOfRunId)) { + runId = run.retryOfRunId; + continue; + } + const [previous] = await db + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, scope.companyId), + ne(heartbeatRuns.id, run.id), + lt(heartbeatRuns.createdAt, run.createdAt), + or( + eq(heartbeatRuns.issueId, scope.issueId), + eq(heartbeatRuns.nativeIssueId, scope.issueId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${scope.issueId}`, + ), + ), + ) + .orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)) + .limit(1); + if (!previous) return { kind: "clear" }; + runId = previous.id; continue; } const payload = record(action.payload); diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index e1ac0519ff0..cd8aee1e94f 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -2465,7 +2465,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende if (countsRecovery || (agedOldest && facts.length > 1)) { const oldestRecoveryAt = countsRecovery ? Math.min(...recoveryRuns.map(entry => entry.createdAt.getTime())) : Infinity; const since = new Date(Math.min(oldestRecoveryAt, agedOldest?.createdAt.getTime() ?? Infinity)); - const recentStarts = await db.select({ wakeReason: sql`${heartbeatRuns.contextSnapshot}->>'wakeReason'`, + const recentStarts = await db.select({ id: heartbeatRuns.id, wakeReason: sql`${heartbeatRuns.contextSnapshot}->>'wakeReason'`, startedAt: heartbeatRuns.startedAt, createdAt: heartbeatRuns.createdAt }) .from(heartbeatRuns) .where(and(eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id), @@ -2475,7 +2475,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende const startsSince = (at: number) => recentStarts.filter(row => row.startedAt && row.startedAt.getTime() >= at); if (countsRecovery) { const starts = startsSince(oldestRecoveryAt).slice(0, RECOVERY_DISPATCH_SHARE); - const lastRecovery = starts.findIndex(row => isRecoveryWakeReason(row.wakeReason)); + const recoveryStartIds = starts.flatMap(row => isRecoveryWakeReason(row.wakeReason) ? [row.id] : []); + const personStarts = new Set(recoveryStartIds.length + ? await listPersonOriginSourceRunIds(agent.companyId, recoveryStartIds) : []); + const lastRecovery = starts.findIndex(row => isRecoveryWakeReason(row.wakeReason) && !personStarts.has(row.id)); startsSinceRecovery = lastRecovery < 0 ? starts.length : lastRecovery; } if (agedOldest && facts.length > 1) {