From 32ee2e9a6a35da4cbde0c7f57df92f4d7eda5444 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Fri, 16 Oct 2026 19:07:50 -0700 Subject: [PATCH] fix(heartbeat): let a live legacy controller finish before the finalization deadline fails its run (#16622) The execution-control reconciler now checks hasLiveLegacyController inside its locked transaction before failing a run. While a live legacy controller lease owns the run, the reconciler moves executionControlDeadlineAt to the lease expiry instead of failing, reporting, or surfacing it. Once the lease lapses, the next sweep fails and reports the run once, as before. Native runs are unchanged. Adds an embedded-Postgres regression test that fails on master and passes with the fix. Co-Authored-By: Claude Opus 5.5 --- .../execution-control-reconciliation.test.ts | 32 +++++++++++++++++++ .../execution-control-reconciliation.ts | 8 +++++ 2 files changed, 40 insertions(+) diff --git a/server/src/services/execution-control-reconciliation.test.ts b/server/src/services/execution-control-reconciliation.test.ts index 292b1795bcf..9d46777715c 100644 --- a/server/src/services/execution-control-reconciliation.test.ts +++ b/server/src/services/execution-control-reconciliation.test.ts @@ -97,6 +97,38 @@ describeEmbeddedPostgres("reconcileAbandonedExecutionControl reports a genuine f }); }); + it("leaves a run owned by a live legacy controller to that controller", async () => { + const { runId } = await seedAbandonedRunFixture(); + const leaseExpiresAt = new Date(Date.now() + 45_000); + await db + .update(heartbeatRuns) + .set({ runtimeMode: "legacy", controllerBootId: randomUUID(), controllerLeaseExpiresAt: leaseExpiresAt }) + .where(eq(heartbeatRuns.id, runId)); + const captureCallsBefore = mockCaptureRunFailure.mock.calls.length; + + const result = await reconcileAbandonedExecutionControl(db); + + expect(result.surfaced).toBe(0); + const [owned] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(owned?.status).toBe("running"); + expect(owned?.errorCode).toBeNull(); + expect(owned?.executionControlDeadlineAt?.getTime()).toBe(leaseExpiresAt.getTime()); + + await db + .update(heartbeatRuns) + .set({ controllerLeaseExpiresAt: new Date(Date.now() - 1_000), executionControlDeadlineAt: new Date(Date.now() - 1_000) }) + .where(eq(heartbeatRuns.id, runId)); + const lapsed = await reconcileAbandonedExecutionControl(db); + + expect(lapsed.surfaced).toBe(1); + const [failed] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(failed?.status).toBe("failed"); + expect(failed?.errorCode).toBe("execution_finalization_deadline_exceeded"); + await vi.waitFor(() => { + expect(mockCaptureRunFailure.mock.calls.slice(captureCallsBefore)).toHaveLength(1); + }, { timeout: 5_000 }); + }); + it("reports zero events for a repeated sweep over the same already-failed run", async () => { const { runId } = await seedAbandonedRunFixture(); await reconcileAbandonedExecutionControl(db); diff --git a/server/src/services/execution-control-reconciliation.ts b/server/src/services/execution-control-reconciliation.ts index b2a0d488dca..72266731dff 100644 --- a/server/src/services/execution-control-reconciliation.ts +++ b/server/src/services/execution-control-reconciliation.ts @@ -10,6 +10,7 @@ import { } from "@paperclipai/db"; import { parseIssueExecutionState } from "./issue-execution-policy.js"; import { issueRecoveryActionService } from "./issue-recovery-actions.js"; +import { hasLiveLegacyController } from "./legacy-controller-lease.js"; import { reportRunFailure } from "./run-failure-report.js"; import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-runtime/native-review-participant.js"; @@ -151,6 +152,13 @@ export async function reconcileAbandonedExecutionControl( .where(eq(nativeRunFinalizations.runId, run.id)); return true; } + if (await hasLiveLegacyController(tx as unknown as Db, run)) { + await tx + .update(heartbeatRuns) + .set({ executionControlDeadlineAt: run.controllerLeaseExpiresAt }) + .where(eq(heartbeatRuns.id, run.id)); + return false; + } const cause = "execution_finalization_deadline_exceeded"; const nextAction = "Inspect the failed run and verify its provider has stopped. Reconcile any uncertain external action before explicitly continuing this task.";