diff --git a/server/src/services/native-runtime/native-restart-recovery.test.ts b/server/src/services/native-runtime/native-restart-recovery.test.ts index 71406202ae5..02fcfd89196 100644 --- a/server/src/services/native-runtime/native-restart-recovery.test.ts +++ b/server/src/services/native-runtime/native-restart-recovery.test.ts @@ -70,6 +70,26 @@ describe("native restart recovery classification", () => { expect(result.allowed).toBe(!live); expect(probe).not.toHaveBeenCalled(); }); + it("takes over a live remote lease at once when its controller is proven gone in this process space", async () => { + const now = new Date(); + const startedAt = new Date(now.getTime() - 60_000); + const owner = { leaseOwner: "old-controller", leaseExpiresAt: new Date(now.getTime() + 50_000), + controllerPid: 424242, controllerProcessStartedAt: startedAt, + controllerHostHash: currentProcessSpaceHash(), controllerBootId: currentControllerBootId() }; + // A hard restart: the previous controller's PID is gone. + await expect(evaluateNativeControllerTakeover({ now, remoteLeaseFenced: true, owner, isProcessAlive: () => false })) + .resolves.toMatchObject({ allowed: true, reason: "remote_lease_controller_process_dead" }); + // The previous controller is still running: keep waiting for its lease. + await expect(evaluateNativeControllerTakeover({ now, remoteLeaseFenced: true, owner, + isProcessAlive: () => true, readProcessStartedAt: async () => startedAt })) + .resolves.toEqual({ allowed: false, reason: "remote_controller_lease_live" }); + // Another process space (a different container or host): never probed, keep waiting. + const probe = vi.fn(() => false); + await expect(evaluateNativeControllerTakeover({ now, remoteLeaseFenced: true, + owner: { ...owner, controllerHostHash: "sha256:another-process-space" }, isProcessAlive: probe })) + .resolves.toEqual({ allowed: false, reason: "remote_controller_lease_live" }); + expect(probe).not.toHaveBeenCalled(); + }); it("does not reopen a failed checkpoint or reset an exhausted provider budget", () => { const evidence = { runnerPidAlive: false, diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 5f9608db4cd..5b07bf46788 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -419,9 +419,24 @@ export async function evaluateNativeControllerTakeover(input: { }): Promise { const { owner, now } = input; if (input.remoteLeaseFenced) { - return owner.leaseExpiresAt && owner.leaseExpiresAt > now - ? { allowed: false, reason: "remote_controller_lease_live" } - : { allowed: true, reason: "remote_controller_lease_released_or_expired" }; + if (!owner.leaseExpiresAt || owner.leaseExpiresAt <= now) { + return { allowed: true, reason: "remote_controller_lease_released_or_expired" }; + } + // The lease fences a second live controller from driving the same remote + // runner. When the lease holder recorded this exact process space, the + // shared prover can show it is gone (a hard restart), so the takeover need + // not wait out the lease. A controller in another process space (another + // container or host) is never probed here: its lease must expire. The + // one-release null-host fallback is never enough for a remote runner. + const sameProcessSpace = Object.prototype.hasOwnProperty.call(owner, "controllerHostHash") + && typeof owner.controllerHostHash === "string" + && owner.controllerHostHash !== UNKNOWN_PROCESS_SPACE + && owner.controllerHostHash === currentProcessProbe().processSpace; + if (sameProcessSpace && owner.controllerPid) { + const local = await evaluateNativeControllerTakeover({ ...input, remoteLeaseFenced: false, requireStoppedController: true }); + if (local.allowed && !local.legacyHostFallback) return { allowed: true, reason: `remote_lease_${local.reason}` }; + } + return { allowed: false, reason: "remote_controller_lease_live" }; } if ((!owner.leaseOwner || !owner.leaseExpiresAt) && !input.requireStoppedController) { return { allowed: true, reason: "unowned" };