diff --git a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts index 8e43a723ef6..71024185b23 100644 --- a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts @@ -8,14 +8,15 @@ import { createHash, randomUUID } from "node:crypto"; import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agentTaskSessions, agents, companies, controllerInstances, createDb, environmentLeases, environments, heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, nativeRunnerCheckpoints } from "@paperclipai/db"; import { DurablePrpControlPlane, parseNativeExecutionInput, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js"; import { startRecoveryPostgres, recoveryPostgresSupport } from "../../__tests__/helpers/recovery-postgres.js"; import { createIsolatedTestHome } from "../../__tests__/helpers/isolated-test-home.js"; import { prepareNativeHeartbeatRun } from "./prepare-native-run.js"; -import { nativeRunnerCheckpointWriter } from "./native-runner-checkpoint.js"; +import { assertNativeRunnerCheckpointOwner, loadNativeRunnerCheckpoint, nativeRunnerCheckpointWriter } from "./native-runner-checkpoint.js"; +import { assertNativeHarnessBackupOwner, nativeHarnessBackupOwner } from "./native-harness-backup-storage.js"; import { assertNativeRetainedSandboxBinding, createNativeControllerRecovery, deferNativeControllerRecovery, deferNativeRunForBusyAgentDirectory, NATIVE_AGENT_DIRECTORY_BUSY_WAIT, importNativeControllerRecovery, NativeControllerRecoveryWaitingError, NATIVE_RETAINED_OWNER_WAIT, resolveNativeControllerRecovery } from "./native-controller-recovery.js"; import { claimNativeSessionResumptions } from "./native-finalization-reconciler.js"; import { claimNativeRestartRecoveries, currentNativeControllerIdentity } from "./native-restart-recovery.js"; @@ -27,6 +28,7 @@ import { inspectRemoteWarmSessionRecovery, waitForLiveWarmSessionOwner } from ". import { currentProcessSpaceHash } from "../controller-process-identity.js"; import { registerControllerInstanceDb, resetControllerInstanceLeaseForTests } from "../controller-instance-lease.js"; import { readProcessStartedAt } from "../hot-restart.js"; +import { PROCESS_START_REQUESTED } from "../native-process-start-events.js"; describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable discovery and import", () => { let database: Awaited>; @@ -610,17 +612,20 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d await new Promise(resolve => oldServer.once("spawn", resolve)); const startedAt = await readProcessStartedAt(oldServer.pid!); expect(startedAt).toBeTruthy(); - const { inspect, recordOwner, runId, current } = await seedWarmFollowUpOnOldServer(); - await recordOwner({ controllerBootId: randomUUID(), controllerPid: oldServer.pid!, controllerHostHash: currentProcessSpaceHash(), + const { inspect, recordOwner, runId, current, fixture } = await seedWarmFollowUpOnOldServer(); + const outgoingBoot = randomUUID(); + await recordOwner({ controllerBootId: outgoingBoot, controllerPid: oldServer.pid!, controllerHostHash: currentProcessSpaceHash(), controllerProcessStartedAt: new Date(startedAt!), leaseOwner: null, leaseExpiresAt: null }); const before = Date.now(); const waiting = await inspect().catch(error => error); expect(waiting).toBeInstanceOf(NativeControllerRecoveryWaitingError); + expect(waiting.owner).toEqual({ runId: fixture.scope.runId, controllerBootId: outgoingBoot }); expect(waiting.retryAt.getTime()).toBeGreaterThan(before); expect(waiting.retryAt.getTime()).toBeLessThanOrEqual(Date.now() + 10_000); - await deferNativeControllerRecovery(db, { ...current.binding, retryAt: waiting.retryAt }); + await deferNativeControllerRecovery(db, { ...current.binding, retryAt: waiting.retryAt, waitingForOwner: waiting.owner }); expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0]) - .toMatchObject({ phase: "retryable_failure", attempt: 0, failureCode: NATIVE_RETAINED_OWNER_WAIT, resultId: null }); + .toMatchObject({ phase: "retryable_failure", attempt: 0, failureCode: NATIVE_RETAINED_OWNER_WAIT, resultId: null, + failureDetail: expect.objectContaining({ waitingForOwner: { runId: fixture.scope.runId, controllerBootId: outgoingBoot } }) }); const [task] = await db.select().from(issues).where(eq(issues.executionRunId, runId)); expect(task?.status).toBe("in_progress"); @@ -685,6 +690,129 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d } finally { resetControllerInstanceLeaseForTests(); } }); + async function parkFollowUpBehindOutgoingServer(waitingFor?: (owner: { runId: string; controllerBootId: string }) => { runId: string; controllerBootId: string }) { + const seeded = await seedWarmFollowUpOnOldServer(); + const { fixture, current, identity, recordOwner, runId } = seeded; + const outgoingBoot = fixture.owner.bootId; + await recordOwner({ controllerBootId: outgoingBoot, controllerPid: 7, controllerHostHash: "outgoing-container", + controllerProcessStartedAt: new Date(Date.now() - 600_000), leaseOwner: null, leaseExpiresAt: null }); + await db.insert(controllerInstances).values({ bootId: outgoingBoot, processSpace: "outgoing-container", pid: 7, + leaseExpiresAt: new Date(Date.now() + 90_000) }); + await db.update(nativeRunFinalizations).set({ controllerBootId: randomUUID() }).where(eq(nativeRunFinalizations.runId, runId)); + const wait = (checkpointedAt: Date | null) => waitForLiveWarmSessionOwner({ db, execution: current, + priorRunId: fixture.scope.runId, checkpointedAt: async () => checkpointedAt }); + const waiting = await wait(null).catch(error => error); + expect(waiting).toBeInstanceOf(NativeControllerRecoveryWaitingError); + expect(waiting.owner).toEqual({ runId: fixture.scope.runId, controllerBootId: outgoingBoot }); + const parkedAt = Date.now(); + await deferNativeControllerRecovery(db, { ...current.binding, retryAt: waiting.retryAt, + waitingForOwner: waitingFor ? waitingFor(waiting.owner) : waiting.owner }); + const outgoingRoot = join(root, `${fixture.scope.runId}-outgoing`); + (await loadNativeRunnerCheckpoint(db, { companyId, issueId: fixture.scope.issueId, runId: fixture.scope.runId }))! + .restoreSessionDirectory(outgoingRoot); + const outgoing = new DurablePrpControlPlane({ stateDirectory: join(outgoingRoot, "control-plane"), identity, + expectedRunnerVersion: "discovery-fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + outgoing.setCheckpointPersistence(nativeRunnerCheckpointWriter(db, fixture.owner)); + const fenceHarnessBackup = async () => assertNativeHarnessBackupOwner(db, await nativeHarnessBackupOwner(db, { + companyId, issueId: fixture.scope.issueId, runId: fixture.scope.runId, bootId: outgoingBoot })); + const saveSuspend = async () => { + outgoing.queueCommand("runner.suspend", {}, `idle-close-${randomUUID()}`); + await outgoing.flushCheckpoint(); + }; + const closeIdleSession = async () => { + await fenceHarnessBackup(); + await saveSuspend(); + return new Date(); + }; + return { ...seeded, wait, waiting: waiting as NativeControllerRecoveryWaitingError, parkedAt, outgoing, + fenceHarnessBackup, saveSuspend, closeIdleSession }; + } + + it("the outgoing server checkpoints and closes its idle warm session while the incoming server's follow-up waits for it, and the follow-up resumes within one recheck", async () => { + registerControllerInstanceDb(db); + const parked = await parkFollowUpBehindOutgoingServer(); + try { + const { fixture, runId, wait, waiting, parkedAt, outgoing, closeIdleSession } = parked; + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0]) + .toMatchObject({ phase: "retryable_failure", attempt: 0, leaseOwner: null, failureCode: NATIVE_RETAINED_OWNER_WAIT, + failureDetail: expect.objectContaining({ waitingForOwner: waiting.owner }) }); + const [prior] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.scope.runId)); + const ownerExitDeadline = prior!.finishedAt!.getTime() + 600_000 + 120_000; + + const checkpointedAt = await closeIdleSession(); + const saved = await loadNativeRunnerCheckpoint(db, { companyId, issueId: fixture.scope.issueId, runId: fixture.scope.runId }); + const restoredRoot = join(root, `${runId}-idle-close`); + saved!.restoreSessionDirectory(restoredRoot); + const restored = JSON.parse(await readFile(join(restoredRoot, "control-plane", "control-plane-state.json"), "utf8")); + expect(restored.commands).toContainEqual(expect.objectContaining({ type: "runner.suspend" })); + await outgoing.stop(); + + expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "incoming", runIds: [runId] })).toEqual([]); + const claims = await claimNativeSessionResumptions({ db, runnerInstanceId: "incoming", runIds: [runId], now: waiting.retryAt }); + expect(claims).toHaveLength(1); + await expect(wait(checkpointedAt)).resolves.toBeUndefined(); + const waitedMs = Math.max(waiting.retryAt.getTime(), checkpointedAt.getTime()) - parkedAt; + expect(waitedMs).toBeLessThanOrEqual(5_000); + expect(ownerExitDeadline - parkedAt).toBeGreaterThan(600_000); + } finally { + await parked.outgoing.stop(); + resetControllerInstanceLeaseForTests(); + } + }); + + it("holds a resumption claim of the parked follow-up until the outgoing server's save commits", async () => { + registerControllerInstanceDb(db); + const parked = await parkFollowUpBehindOutgoingServer(); + try { + const { fixture, runId, waiting } = parked; + let claim!: Promise>>; + let claimSettled = false; + await db.transaction(async tx => { + await assertNativeRunnerCheckpointOwner(tx, fixture.owner, false); + claim = claimNativeSessionResumptions({ db, runnerInstanceId: "incoming", runIds: [runId], now: waiting.retryAt }); + void claim.finally(() => { claimSettled = true; }); + await vi.waitFor(async () => { + const [blocked] = await db.execute<{ count: number }>(sql`select count(*)::int as count from pg_stat_activity + where datname = current_database() and wait_event_type = 'Lock' and query ilike '%native_run_finalizations%'`); + expect(blocked.count).toBeGreaterThan(0); + }, { timeout: 1_500, interval: 20 }); + expect(claimSettled).toBe(false); + }); + expect(await claim).toHaveLength(1); + await expect(parked.saveSuspend()).rejects.toThrow("native_runner_checkpoint_owner_lost"); + } finally { + await parked.outgoing.stop(); + resetControllerInstanceLeaseForTests(); + } + }); + + it.each(["own-process", "own-process-event", "own-checkpoint", "claimed", "other-owner-boot", "other-owner-run"] as const)( + "refuses the outgoing server's idle checkpoint when the parked follow-up is no longer waiting on it: %s", async scenario => { + registerControllerInstanceDb(db); + const parked = await parkFollowUpBehindOutgoingServer(owner => scenario === "other-owner-boot" + ? { ...owner, controllerBootId: randomUUID() } : scenario === "other-owner-run" ? { ...owner, runId: randomUUID() } : owner); + try { + const { fixture, runId, waiting, fenceHarnessBackup, saveSuspend } = parked; + if (scenario === "own-process") { + await db.update(heartbeatRuns).set({ processPid: 4242, processGroupId: 4242, processStartedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + } else if (scenario === "own-process-event") { + await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 1, eventType: PROCESS_START_REQUESTED }); + } else if (scenario === "own-checkpoint") { + await db.insert(nativeRunnerCheckpoints).values({ companyId, issueId: fixture.scope.issueId, runId, part: "header", material: {} }); + } else if (scenario === "claimed") { + expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "incoming", runIds: [runId], now: waiting.retryAt })).toHaveLength(1); + } + const before = await loadNativeRunnerCheckpoint(db, { companyId, issueId: fixture.scope.issueId, runId: fixture.scope.runId }); + await expect(fenceHarnessBackup()).rejects.toThrow("native_runner_checkpoint_owner_lost"); + await expect(saveSuspend()).rejects.toThrow("native_runner_checkpoint_owner_lost"); + expect((await loadNativeRunnerCheckpoint(db, { companyId, issueId: fixture.scope.issueId, runId: fixture.scope.runId }))?.records) + .toEqual(before?.records); + } finally { + await parked.outgoing.stop(); + resetControllerInstanceLeaseForTests(); + } + }); + it("sends a re-wake after a crash restart to a fresh sandbox when the parked predecessor's warm Runner cannot be adopted", async () => { const fixture = await seed(), runId = randomUUID(), environmentId = randomUUID(), leaseId = randomUUID(); const previous = parseNativeExecutionInput({ diff --git a/server/src/services/native-runtime/native-controller-recovery.ts b/server/src/services/native-runtime/native-controller-recovery.ts index a5696d40f02..c7523d642f0 100644 --- a/server/src/services/native-runtime/native-controller-recovery.ts +++ b/server/src/services/native-runtime/native-controller-recovery.ts @@ -70,8 +70,10 @@ export type NativeControllerRecoveryOutcome = export const NATIVE_RETAINED_OWNER_WAIT = "native_controller_recovery_waiting_for_owner"; +export type NativeRetainedOwnerWait = { runId: string; controllerBootId: string }; + export class NativeControllerRecoveryWaitingError extends Error { - constructor(readonly retryAt: Date) { + constructor(readonly retryAt: Date, readonly owner?: NativeRetainedOwnerWait) { super(NATIVE_RETAINED_OWNER_WAIT); this.name = "NativeControllerRecoveryWaitingError"; } @@ -83,7 +85,7 @@ export class NativeControllerRecoveryWaitingError extends Error { * permissions and task ownership when this deadline arrives. */ export async function deferNativeControllerRecovery(db: Db, input: { companyId: string; agentId: string; issueId: string; runId: string; - leaseOwner?: string; retryAt: Date; + leaseOwner?: string; retryAt: Date; waitingForOwner?: NativeRetainedOwnerWait; }): Promise { await db.transaction(async tx => { const [task] = await tx.select({ executionRunId: issues.executionRunId }).from(issues) @@ -128,7 +130,9 @@ export async function deferNativeControllerRecovery(db: Db, input: { failureDetail: { originalFailureCode: NATIVE_RETAINED_OWNER_WAIT, message: "Waiting for the retained native controller owner to release its lease.", recoveryMode: record(run.runnerProfileJson).sessionCheckpoint ? "same_session_resume" : "bootstrap_retry", - providerSessionEstablished: false, providerEventsExist: false }, + providerSessionEstablished: false, providerEventsExist: false, + ...(input.waitingForOwner ? { waitingForOwner: { runId: input.waitingForOwner.runId, + controllerBootId: input.waitingForOwner.controllerBootId } } : {}) }, updatedAt: new Date(), }).where(eq(nativeRunFinalizations.runId, input.runId)); }); diff --git a/server/src/services/native-runtime/native-runner-checkpoint.ts b/server/src/services/native-runtime/native-runner-checkpoint.ts index 055168d68e5..3a76c84ac17 100644 --- a/server/src/services/native-runtime/native-runner-checkpoint.ts +++ b/server/src/services/native-runtime/native-runner-checkpoint.ts @@ -3,12 +3,14 @@ import { NATIVE_PREPARATION_FENCING_VERSION } from "./native-active-runs.js"; import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js"; import { createHash } from "node:crypto"; import { and, eq, exists, gt, isNull, lte, inArray, notExists, or, sql, type SQL } from "drizzle-orm"; -import { checkpointDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, primeDatabaseWorkTransactionId, type Db } from "@paperclipai/db"; +import { checkpointDb, heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, primeDatabaseWorkTransactionId, type Db } from "@paperclipai/db"; import { inspectDurablePrpCheckpointHeader, inspectUnauthenticatedBootstrapCheckpoint, inspectUnclaimedBootstrapCheckpoint, decodeDurablePrpCheckpoint, type DurablePrpCheckpoint, type DurablePrpControlPlane, type DurableRecoveryIdentity } from "../../vendor/paperclip-runner/index.js"; import { createLocalEncryptedCipher, deriveLocalEncryptedInstanceSecret } from "../../secrets/local-encrypted-provider.js"; import { legacyControllerBootId } from "../legacy-controller-lease.js"; import { assertNativeSourceCheckpointConsumer, lockSourceRecoveryCoordinators } from "./native-source-recovery-ownership.js"; import { runnerdStateHasNoProviderTurn, runnerdStateIsIncompleteBootstrap } from "./native-bootstrap-proof.js"; +import { NATIVE_RETAINED_OWNER_WAIT } from "./native-controller-recovery.js"; +import { OWN_PROCESS_START_EVENT_TYPES } from "../native-process-start-events.js"; type Binding = { companyId: string; issueId: string; runId: string }; export const NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES = 1024 * 1024; @@ -116,13 +118,37 @@ export async function validateNativeRunnerCheckpointOwner( eq(heartbeatRuns.issueId, owner.issueId), inArray(heartbeatRuns.status, ["queued", "scheduled_retry"]), notExists(tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations) .where(eq(nativeRunFinalizations.runId, heartbeatRuns.id))))) : []; + const [parkedSuccessor] = !successor && !preparingSuccessor && !waitingSuccessor && task?.executionRunId && task.executionRunId !== owner.runId + ? await parkedSuccessorWaitingForOwner(tx, owner, task.executionRunId) : []; if (!current.resultId || newWork || !task - || (task.executionRunId && task.executionRunId !== owner.runId && !successor && !preparingSuccessor && !waitingSuccessor)) { + || (task.executionRunId && task.executionRunId !== owner.runId && !successor && !preparingSuccessor && !waitingSuccessor && !parkedSuccessor)) { throw new Error("native_runner_checkpoint_owner_lost"); } } } +function parkedSuccessorWaitingForOwner(tx: CheckpointTransaction, owner: NativeRunnerCheckpointFenceOwner, successorRunId: string) { + return tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations) + .innerJoin(heartbeatRuns, eq(heartbeatRuns.id, nativeRunFinalizations.runId)).where(and( + eq(nativeRunFinalizations.runId, successorRunId), eq(nativeRunFinalizations.companyId, owner.companyId), + eq(nativeRunFinalizations.issueId, owner.issueId), eq(nativeRunFinalizations.phase, "retryable_failure"), + eq(nativeRunFinalizations.failureCode, NATIVE_RETAINED_OWNER_WAIT), eq(nativeRunFinalizations.attempt, 0), + isNull(nativeRunFinalizations.resultId), isNull(nativeRunFinalizations.leaseOwner), + sql`${nativeRunFinalizations.failureDetail}->'waitingForOwner'->>'runId' = ${owner.runId}`, + sql`${nativeRunFinalizations.failureDetail}->'waitingForOwner'->>'controllerBootId' = ${owner.bootId}`, + eq(heartbeatRuns.companyId, owner.companyId), eq(heartbeatRuns.nativeIssueId, owner.issueId), + eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.status, "running"), + isNull(heartbeatRuns.processPid), isNull(heartbeatRuns.processGroupId), isNull(heartbeatRuns.processStartedAt), + sql`${heartbeatRuns.nativeSessionId} = (select source.native_session_id from heartbeat_runs source + where source.id = ${owner.runId} and source.company_id = ${owner.companyId})`, + notExists(tx.select({ runId: nativeRunnerCheckpoints.runId }).from(nativeRunnerCheckpoints).where(and( + eq(nativeRunnerCheckpoints.companyId, owner.companyId), eq(nativeRunnerCheckpoints.runId, successorRunId)))), + notExists(tx.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and( + eq(heartbeatRunEvents.companyId, owner.companyId), eq(heartbeatRunEvents.runId, successorRunId), + isNull(heartbeatRunEvents.sourceEventId), inArray(heartbeatRunEvents.eventType, [...OWN_PROCESS_START_EVENT_TYPES])))), + )).for("share", { of: nativeRunFinalizations }); +} + export async function assertNativeRunnerCheckpointOwner(tx: CheckpointTransaction, owner: NativeRunnerCheckpointOwner, newWork: boolean) { const locked = await lockNativeRunnerCheckpointOwner(tx, owner); await validateNativeRunnerCheckpointOwner(tx, owner, locked, newWork); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index c78ad91a900..a225db8c5a7 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -10735,7 +10735,7 @@ export async function executePaperclipNativeSession(input: { executionFailure = error; if (error instanceof NativeControllerRecoveryWaitingError) { await deferNativeControllerRecovery(input.db, { ...input.execution.binding, - leaseOwner: input.leaseOwner, retryAt: error.retryAt }); + leaseOwner: input.leaseOwner, retryAt: error.retryAt, waitingForOwner: error.owner }); } if (!(error instanceof NativeRunnerOwnershipUnverifiedError) && !(error instanceof NativeStartupCleanupPendingError) && startup.stopRequested && startup.executionContext?.signal.reason instanceof Error && startup.executionContext.signal.reason.message === "native_session_cancelled") { diff --git a/server/src/services/native-runtime/remote-warm-session-recovery.ts b/server/src/services/native-runtime/remote-warm-session-recovery.ts index 113cb8b5be9..3397917819e 100644 --- a/server/src/services/native-runtime/remote-warm-session-recovery.ts +++ b/server/src/services/native-runtime/remote-warm-session-recovery.ts @@ -53,10 +53,10 @@ export async function waitForLiveWarmSessionOwner(input: { const finishedAt = prior?.finishedAt ?? null; const retryAt = await liveWarmSessionOwnerRetryAt({ owner, finishedAt, idleTimeoutMs: input.execution.session.lifecyclePolicy.idleTimeoutMs ?? null }); - if (!retryAt) return; + if (!retryAt || !owner.controllerBootId) return; const checkpointedAt = await input.checkpointedAt(); if (checkpointedAt && finishedAt && checkpointedAt >= finishedAt) return; - throw new NativeControllerRecoveryWaitingError(retryAt); + throw new NativeControllerRecoveryWaitingError(retryAt, { runId: input.priorRunId, controllerBootId: owner.controllerBootId }); } export type RemoteWarmSessionRecovery = { @@ -228,7 +228,9 @@ export async function inspectRemoteWarmSessionRecovery(input: { if (prior.owner.phase !== "committed" || prior.owner.leaseOwner) return deny("owner_unsettled"); const liveOwnerRetryAt = await liveWarmSessionOwnerRetryAt({ owner: prior.owner, finishedAt: prior.run.finishedAt, idleTimeoutMs: previous.session?.lifecyclePolicy?.idleTimeoutMs ?? null }); - if (liveOwnerRetryAt) throw new NativeControllerRecoveryWaitingError(liveOwnerRetryAt); + if (liveOwnerRetryAt) { + throw new NativeControllerRecoveryWaitingError(liveOwnerRetryAt, { runId: identity.runId, controllerBootId: prior.owner.controllerBootId }); + } if (sourceProviderLeaseId === providerLeaseId) settledPredecessor = { runId: identity.runId, providerLeaseId }; const checkpoint = input.loadCheckpoint(previous); if (!checkpoint || checkpoint.identity.runId !== identity.runId ||