diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index d402cdebe86..4f9859d9156 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -584,13 +584,26 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { return { environmentId, leaseId }; } - async function seedDeadRunFinalizeBarrierFixture(input?: { runStatus?: "running" | "failed" }) { + async function seedDeadRunFinalizeBarrierFixture(input?: { + runStatus?: "running" | "interrupted" | "failed"; + }) { + const runStatus = input?.runStatus ?? "running"; + const runErrorCode = runStatus === "failed" + ? "process_lost" + : runStatus === "interrupted" + ? "server_shutdown_interrupted" + : null; + const runError = runStatus === "failed" + ? "Process lost during a prior server lifetime" + : runStatus === "interrupted" + ? "Run interrupted during graceful server shutdown" + : null; const seeded = await seedRunFixture({ agentStatus: "running", adapterType: "test", - runStatus: input?.runStatus ?? "running", - runErrorCode: input?.runStatus === "failed" ? "process_lost" : null, - runError: input?.runStatus === "failed" ? "Process lost during a prior server lifetime" : null, + runStatus, + runErrorCode, + runError, }); const projectId = randomUUID(); const executionWorkspaceId = randomUUID(); @@ -631,7 +644,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { issueId: seeded.issueId, executionWorkspaceId, }, - ...(input?.runStatus === "failed" ? { finishedAt: now, updatedAt: now } : {}), + ...(runStatus !== "running" ? { finishedAt: now, updatedAt: now } : {}), }) .where(eq(heartbeatRuns.id, seeded.runId)); @@ -1489,6 +1502,41 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }); }); + it("releases a finalize barrier owned by an interrupted run", async () => { + const fixture = await seedDeadRunFinalizeBarrierFixture({ runStatus: "interrupted" }); + + const result = await heartbeatService(db).reconcileTerminalRunWorkspaceFinalizes("startup_sweep"); + + expect(result).toMatchObject({ + checked: 1, + created: 1, + wakesHealed: 1, + runIds: [fixture.runId], + }); + const syntheticFinalize = await db + .select() + .from(workspaceOperations) + .where( + and( + eq(workspaceOperations.heartbeatRunId, fixture.runId), + eq(workspaceOperations.phase, "workspace_finalize"), + ), + ) + .then((rows) => rows[0] ?? null); + expect(syntheticFinalize).toMatchObject({ + status: "succeeded", + metadata: expect.objectContaining({ + synthetic: true, + reason: "server_shutdown_interrupted", + terminalRunStatus: "interrupted", + }), + }); + await expect(issueService(db).getDependencyReadiness(fixture.dependentIssueId)).resolves.toMatchObject({ + isDependencyReady: true, + pendingFinalizeBlockerIssueIds: [], + }); + }); + it("does not mask an explicit failed workspace finalize with a synthetic success", async () => { const fixture = await seedDeadRunFinalizeBarrierFixture({ runStatus: "failed" }); const failedAt = new Date("2026-03-19T00:02:00.000Z"); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index d9b9c752e67..69774c78e53 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -291,6 +291,10 @@ const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_r const CANCELLABLE_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const; const HEARTBEAT_RUN_TERMINAL_STATUSES = ["succeeded", "interrupted", "failed", "cancelled", "timed_out"] as const; const UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES = ["failed", "cancelled", "timed_out"] as const; +const WORKSPACE_FINALIZE_RECOVERABLE_TERMINAL_STATUSES = [ + "interrupted", + ...UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES, +] as const; const TIMER_ACTIONABLE_ISSUE_STATUSES = ["todo", "in_progress"] as const; export { ACTIVE_RUN_OUTPUT_CONTINUE_REARM_MS, @@ -5102,8 +5106,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const reconciliation = (async () => { if ( - !UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES.includes( - run.status as (typeof UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES)[number], + !WORKSPACE_FINALIZE_RECOVERABLE_TERMINAL_STATUSES.includes( + run.status as (typeof WORKSPACE_FINALIZE_RECOVERABLE_TERMINAL_STATUSES)[number], ) ) { return { created: false, wakeHealed: 0 }; @@ -5306,7 +5310,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) and( eq(heartbeatRuns.id, latestOperation.heartbeatRunId), eq(heartbeatRuns.companyId, blocker.companyId), - inArray(heartbeatRuns.status, [...UNSUCCESSFUL_HEARTBEAT_RUN_TERMINAL_STATUSES]), + inArray(heartbeatRuns.status, [...WORKSPACE_FINALIZE_RECOVERABLE_TERMINAL_STATUSES]), ), ) .then((rows) => rows[0] ?? null);