diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index a860dae7cc9..bd43b03ce1c 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -110,6 +110,7 @@ import { shouldQueueFollowupForRunningIssueWake, DEFERRED_WAKE_CONTEXT_KEY, createHeartbeatQueue, + notifyOtherAgentsQueuedOnSettledIssue, unsavedWakeAdmissionCount, } from "./heartbeat/queue.js"; export { @@ -4321,6 +4322,11 @@ export function heartbeatService( }); } await startNextQueuedRunForAgent(run.agentId); + if (latestRun && isHeartbeatRunTerminalStatus(latestRun.status)) { + await notifyOtherAgentsQueuedOnSettledIssue(db, latestRun).catch(err => { + logger.warn({ err, runId: run.id }, "failed to wake the task's next owner after execution cleanup"); + }); + } } } } diff --git a/server/src/services/heartbeat/queue.test.ts b/server/src/services/heartbeat/queue.test.ts index 168b0558bf6..e54dd59825f 100644 --- a/server/src/services/heartbeat/queue.test.ts +++ b/server/src/services/heartbeat/queue.test.ts @@ -10,7 +10,7 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } fro import { createPostgresWakeQueueAdapter, createWakeAdmissionWriter } from "../../modules/wake-queue/adapters/postgres.js"; import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js"; import { closeProcessRunAdmission, resetProcessRunAdmissionForTests } from "./run-admission.js"; -import { createHeartbeatQueue, DEFERRED_WAKE_CONTEXT_KEY, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake, type HeartbeatQueueDependencies } from "./queue.js"; +import { createHeartbeatQueue, DEFERRED_WAKE_CONTEXT_KEY, notifyOtherAgentsQueuedOnSettledIssue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake, type HeartbeatQueueDependencies } from "./queue.js"; import { createWakeQueue } from "../../modules/wake-queue/index.js"; import { TransactionScope } from "../../modules/wake-queue/application/ports.js"; import { enrichWakeContextSnapshot, mergeCoalescedContextSnapshot } from "./run-preparation.js"; @@ -22,6 +22,7 @@ import { createHeartbeatRunControl } from "./run-control.js"; import { callbacks as runControlCallbacks } from "./__tests__/run-control-fixtures.js"; import { createHeartbeatQueueTestDependencies as callbacks } from "../../__tests__/helpers/heartbeat-queue.js"; +import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../delivery-work-notifications.js"; function guardedDatabase() { const access = vi.fn(() => { throw new Error("Unexpected queue database access"); }); @@ -634,6 +635,56 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { expect(await queue.claimQueuedRun(successor)).toBeNull(); }); + async function reassignedPredecessorFixture(previousStatus = "cancelled") { + const f = await fixture("queued"); + await db.update(issues).set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null }) + .where(eq(issues.id, f.issue.id)); + const [previousAgent] = await db.insert(agents).values({ companyId: f.company.id, name: "Previous owner", + role: "engineer", status: "idle", adapterType: "paperclip_runner", adapterConfig: {} }).returning(); + const [previous] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: previousAgent.id, + status: previousStatus, runtimeMode: "native", invocationSource: "assignment", startedAt: new Date(), + finishedAt: previousStatus === "running" ? null : new Date(), nativeIssueId: f.issue.id, + errorCode: previousStatus === "cancelled" ? "issue_reassigned" : null, + contextSnapshot: { issueId: f.issue.id, wakeReason: "issue_assigned" } }).returning(); + const [next] = await db.update(heartbeatRuns).set({ responsibleUserId: "queue-test-user", + contextSnapshot: { issueId: f.issue.id, wakeReason: "issue_assigned" } }) + .where(eq(heartbeatRuns.id, f.run.id)).returning(); + const deps = callbacks(db); + deps.getAgent.mockResolvedValue(f.agent); + deps.activeRunExecutions.add(previous.id); + return { ...f, previousAgent, previous, next, deps, queue: createHeartbeatQueue(db, deps) }; + } + + it("keeps the new owner's run queued while the reassigned owner's stopped run still settles", async () => { + const f = await reassignedPredecessorFixture(); + expect(await f.queue.claimQueuedRun(f.next)).toBeNull(); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.next.id)))[0]) + .toMatchObject({ status: "queued", startedAt: null }); + expect(f.deps.executeRun).not.toHaveBeenCalled(); + + f.deps.activeRunExecutions.delete(f.previous.id); + expect(await f.queue.claimQueuedRun(f.next)).toMatchObject({ id: f.next.id, status: "running" }); + }); + + it("does not hold a run behind another agent's live run on the same task", async () => { + const f = await reassignedPredecessorFixture("running"); + expect(await f.queue.claimQueuedRun(f.next)).toMatchObject({ id: f.next.id, status: "running" }); + }); + + it("wakes queued dispatch for the task's next owner once the previous owner's execution settles", async () => { + const f = await reassignedPredecessorFixture(); + let wakes = 0; + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.queuedRunDispatch, () => { wakes += 1; }); + try { + expect(await notifyOtherAgentsQueuedOnSettledIssue(db, f.previous)).toBe(true); + await vi.waitFor(() => expect(wakes).toBeGreaterThan(0)); + await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.next.id)); + expect(await notifyOtherAgentsQueuedOnSettledIssue(db, f.previous)).toBe(false); + } finally { + unsubscribe(); + } + }); + it("keeps a retry queued after failed cleanup and until its local executor releases", async () => { const f = await pendingPredecessorFixture(); await db.update(environmentLeases).set({ status: "released", releasedAt: new Date(), cleanupStatus: "failed" }) diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index e55b996a0aa..fe4adc984b1 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -15,7 +15,7 @@ import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { PersistedExecutionWaitRefusal, writeDeferredExecutionWait } from "../deferred-execution-wait.js"; import { assignmentWakePayloadSql, isAssignmentWakePayload, notifyDeferredExecutionWork } from "../deferred-execution-notifications.js"; import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; -import { notifyRunDispatch } from "../run-dispatch-notifications.js"; +import { notifyQueuedRunDispatch, notifyRunDispatch } from "../run-dispatch-notifications.js"; import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; @@ -200,6 +200,7 @@ import type { createHeartbeatRunState } from "./run-state.js"; import type { createHeartbeatRunPreparation } from "./run-preparation.js"; import type { HeartbeatRetryDependencies } from "./retries.js"; import type { HeartbeatRecoveryDependencies, createHeartbeatRecovery } from "./recovery.js"; +import { HEARTBEAT_RUN_TERMINAL_STATUSES } from "./recovery.js"; import { executeIssuePostCommitActions, type IssuePostCommitAction, type issueService } from "../issues.js"; import type { issueTreeControlService } from "../issue-tree-control.js"; import type { budgetService } from "../budgets.js"; @@ -418,6 +419,21 @@ class RunAdmissionClosedDuringClaim extends Error {} const unsavedWakeAdmissions = new Set(); export function unsavedWakeAdmissionCount() { return unsavedWakeAdmissions.size; } +export async function notifyOtherAgentsQueuedOnSettledIssue(db: Db, run: Pick): Promise { + const issueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); + if (run.runtimeMode !== "native" || !issueId) return false; + const [waiting] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.status, "queued"), + ne(heartbeatRuns.agentId, run.agentId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, + )).limit(1); + if (!waiting) return false; + await db.transaction(tx => notifyQueuedRunDispatch(tx, { companyId: run.companyId })); + return true; +} + export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDependencies) { const { setRunStatusFromLive, @@ -1171,7 +1187,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende .from(heartbeatRuns) .where(and( eq(heartbeatRuns.companyId, run.companyId), - eq(heartbeatRuns.agentId, run.agentId), + or(eq(heartbeatRuns.agentId, run.agentId), inArray(heartbeatRuns.status, [...HEARTBEAT_RUN_TERMINAL_STATUSES])), eq(heartbeatRuns.runtimeMode, "native"), inArray(heartbeatRuns.id, [...activeRunExecutions]), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,