diff --git a/doc/task-recovery.md b/doc/task-recovery.md index 3aecdb16b11..87f69bbb929 100644 --- a/doc/task-recovery.md +++ b/doc/task-recovery.md @@ -11,7 +11,10 @@ It requires authenticated board task-mutation authority. It records an `agent_wakeup_requests` receipt before physical work. Company/key uniqueness covers completed and failed receipts as well as pending ones. Concurrent clicks attach to accepted work. A new deliberate click after exhaustion can create a -new generation; a repeated HTTP request cannot. +new generation; a repeated HTTP request cannot. A click does not renew an +exhausted generation that already requires an alternative strategy and has no +fresh attempt. The click binds to that generation and continues in a fresh +isolated workspace, so it does not repeat the failed step. The existing exact-run retry endpoint retains its source-bound contract. Connected conversations use their existing inbound authorization owner. A board @@ -118,6 +121,16 @@ environment, allocation, stop-only intent, and physical ownership claim before calling the provider. The retained-source fence still blocks ordinary resume and destruction. Recovery does not clear it to gain access to the saved files. +The reopen moves the original stop receipt into the stop-only intent before it +calls the provider. The lease shows no receipt while the reopen is in progress. +When the provider returns the reopened allocation, recovery discards the old +receipt, and the allocation needs a fresh stop. If the reopen fails, recovery +asks the provider to stop the allocation. If that stop also fails, recovery +restores the original receipt and lease state, so a stopped source never stays +without a receipt. A reopen that fails because the sandbox disk is full ends the +generation with `source_unavailable`. Saved input and Retry then continue in a +fresh isolated workspace. + Execution containment and file preservation are independent facts: - Another agent needs authoritative containment of the old executor. diff --git a/server/src/services/sandbox-stop-and-retain.ts b/server/src/services/sandbox-stop-and-retain.ts index d8331c96c28..0bacfbe699d 100644 --- a/server/src/services/sandbox-stop-and-retain.ts +++ b/server/src/services/sandbox-stop-and-retain.ts @@ -97,16 +97,61 @@ export async function prepareHistoricalSandboxStopVerification(db: Db, lease: ty export async function prepareStoppedSandboxForRecovery(db: Db, lease: typeof environmentLeases.$inferSelect) { if (!hasRemoteTerminationReceipt(lease) || lease.status !== "released" || (lease.metadata?.remoteExecutionTermination as { state?: string })?.state !== "stopped") return null; - return prepareStopAndRetain(db, lease, "released"); + const { remoteExecutionTermination, sandboxStopAndRetainReceipt } = lease.metadata!; + return prepareStopAndRetain(db, lease, "released", { priorStop: { remoteExecutionTermination, + ...(sandboxStopAndRetainReceipt === undefined ? {} : { sandboxStopAndRetainReceipt }) } }); } -async function prepareStopAndRetain(db: Db, lease: Lease, fromStatus: "active" | "released") { +export async function confirmSandboxRecoveryRestart(db: Db, lease: typeof environmentLeases.$inferSelect, options: { attemptId: string }) { + if (!suspendedSandboxStop(lease)) return lease; + const [updated] = await db.update(environmentLeases).set({ updatedAt: new Date(), + metadata: sql`jsonb_set(${environmentLeases.metadata}, '{sandboxStopAndRetain}', (${environmentLeases.metadata}->'sandboxStopAndRetain') - 'priorStop')`, + }).where(stopOnlyAttempt(lease, options.attemptId)).returning(); + return updated ?? null; +} + +export async function restoreSandboxStopAfterFailedRestart(db: Db, lease: typeof environmentLeases.$inferSelect, options: { attemptId: string }) { + const prior = suspendedSandboxStop(lease); + if (!prior) return null; + const restored = { remoteExecutionTermination: prior.remoteExecutionTermination, + ...(prior.sandboxStopAndRetainReceipt === undefined ? {} : { sandboxStopAndRetainReceipt: prior.sandboxStopAndRetainReceipt }) }; + if ((restored.remoteExecutionTermination as { state?: unknown } | undefined)?.state !== "stopped" + || !hasRemoteTerminationReceipt({ ...lease, status: "released", cleanupStatus: "success", metadata: { ...lease.metadata, ...restored } })) return null; + const now = new Date(); + const [updated] = await writeEnvironmentCleanup(db, async tx => { + await notifyRetainedNativeSessionCleanup(tx, { companyId: lease.companyId, runId: lease.heartbeatRunId! }); + return tx.update(environmentLeases).set({ + status: "released", cleanupStatus: "success", failureReason: null, updatedAt: now, + metadata: sql`(${environmentLeases.metadata} - 'remoteExecutionTermination' - 'sandboxStopAndRetain' - 'sandboxStopAndRetainReceipt') || ${JSON.stringify({ + ...restored, pendingCleanupInFlight: false, pendingCleanupLeaseExpiresAtMs: 0, + })}::jsonb`, + }).where(stopOnlyAttempt(lease, options.attemptId)).returning(); + }, { companyId: lease.companyId, runId: lease.heartbeatRunId ?? undefined }); + return updated ?? null; +} + +function suspendedSandboxStop(lease: typeof environmentLeases.$inferSelect) { + if (hasNativeWorkspaceExportResume(lease) || !readStopOnlyCleanup(lease)) return null; + const prior = (lease.metadata![SANDBOX_STOP_AND_RETAIN_KEY] as Record).priorStop; + return prior && typeof prior === "object" && !Array.isArray(prior) ? prior as Record : null; +} + +function stopOnlyAttempt(lease: Lease, attemptId: string) { + return and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId), + eq(environmentLeases.heartbeatRunId, lease.heartbeatRunId!), eq(environmentLeases.provider, lease.provider!), + eq(environmentLeases.providerLeaseId, lease.providerLeaseId!), eq(environmentLeases.status, "pending_cleanup"), + sql`${environmentLeases.metadata}->>'pendingCleanupAttemptId' = ${attemptId}`, + sql`${environmentLeases.metadata}->'sandboxStopAndRetain' = ${JSON.stringify(lease.metadata?.[SANDBOX_STOP_AND_RETAIN_KEY])}::jsonb`, + sql`${environmentLeases.metadata}->>'pluginId' is not distinct from ${readStopOnlyCleanup(lease)?.pluginId ?? null}`); +} + +async function prepareStopAndRetain(db: Db, lease: Lease, fromStatus: "active" | "released", recovery: Record = {}) { const requestId = randomUUID(), now = new Date(); const intent = { schema: "paperclip.sandbox-stop-and-retain.v1", requestId, companyId: lease.companyId, runId: lease.heartbeatRunId, leaseId: lease.id, provider: lease.provider, providerLeaseId: lease.providerLeaseId, ...(lease.provider && isBuiltinSandboxProvider(lease.provider) && !lease.metadata?.sandboxProviderPlugin && lease.metadata?.pluginId == null - ? { builtinProvider: lease.provider } : { pluginId: lease.metadata?.pluginId }) }; + ? { builtinProvider: lease.provider } : { pluginId: lease.metadata?.pluginId }), ...recovery }; const [updated] = await writeEnvironmentCleanup(db, tx => tx.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed", failureReason: "sandbox_stop_pending", releasedAt: now, updatedAt: now, metadata: sql`(coalesce(${environmentLeases.metadata}, '{}'::jsonb) - 'remoteExecutionTermination' - 'sandboxStopAndRetainReceipt') || ${JSON.stringify({ @@ -147,13 +192,7 @@ export async function settleStopOnlyCleanup(db: Db, lease: Lease, options: { att } } : {}), pendingCleanupInFlight: false, pendingCleanupLeaseExpiresAtMs: 0, })}::jsonb`, - }).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId), - eq(environmentLeases.heartbeatRunId, lease.heartbeatRunId!), eq(environmentLeases.provider, lease.provider!), - eq(environmentLeases.providerLeaseId, lease.providerLeaseId!), eq(environmentLeases.status, "pending_cleanup"), - sql`${environmentLeases.metadata}->>'pendingCleanupAttemptId' = ${options.attemptId}`, - sql`${environmentLeases.metadata}->'sandboxStopAndRetain' = ${JSON.stringify(lease.metadata?.[SANDBOX_STOP_AND_RETAIN_KEY])}::jsonb`, - sql`${environmentLeases.metadata}->>'pluginId' is not distinct from ${intent.pluginId}`, - )).returning(); + }).where(stopOnlyAttempt(lease, options.attemptId)).returning(); }, { companyId: lease.companyId, runId: lease.heartbeatRunId ?? undefined }); return updated ?? null; } diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index 4b07a2fcb5f..3762e042dcc 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -368,7 +368,8 @@ function admitTaskRetry(input: TaskRetryRequest) { // A completed preservation still owns its snapshot evidence. Bind the // new intent so sibling holds can select that same continuation, without // repeating recovery or renewing its attempt budget. - if (previous.state === "complete" || previous.state === "running") { + if (previous.state === "complete" || previous.state === "running" || (previous.state === "exhausted" + && previous.failure?.retryClass === "use_alternative_strategy" && !previous.freshAttemptRunId)) { await updateSystemRecovery(db, { companyId: issue.companyId, issueId: issue.id, runId: previous.sourceRunId, actionId: action.id }, previous, { ...previous, intentId: active?.id ?? id }); continue; diff --git a/server/src/services/workspace-recovery.test.ts b/server/src/services/workspace-recovery.test.ts index 91d3ad20480..dde9ded92b5 100644 --- a/server/src/services/workspace-recovery.test.ts +++ b/server/src/services/workspace-recovery.test.ts @@ -65,7 +65,7 @@ import { startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded import { nativeWorkspaceSyncStateRoot, prepareNativeWorkspaceSync, resumeNativeWorkspaceSync } from "./native-runtime/native-workspace-sync.js"; import { nativeWorkspaceExportPendingForResource } from "./native-runtime/native-workspace-finalization-state.js"; import { nativeWorkspaceFinalizationLockKey } from "./native-runtime/native-workspace-finalization-ownership.js"; -import { remoteTerminationReceipt } from "./remote-execution-termination.js"; +import { hasRemoteTerminationReceipt, remoteTerminationReceipt } from "./remote-execution-termination.js"; import { prepareStoppedSandboxForRecovery, prepareSandboxStopAndRetain, settleStopOnlyCleanup, readStopOnlyCleanup } from "./sandbox-stop-and-retain.js"; import { recoverRetainedWorkspace, requestWorkspaceRecovery, scheduleRecoveredWorkspaceCleanup, reconcileRetainedWorkspaces, retainedWorkspaceCandidates } from "./workspace-recovery.js"; import * as activity from "./activity-log.js"; @@ -3520,6 +3520,100 @@ describe("automatic retained workspace recovery", () => { expect(after.revision).toBeGreaterThan(before.revision); }, 30_000); + const restartFailure = (diskFull: boolean) => new Error(diskFull + ? "Sandbox failed to start with state: error, error reason: write /home/daytona/workspace/.git/index: no space left on device" + : "Sandbox failed to start with state: error"); + const leaseRow = async (id: string) => (await db.select().from(environmentLeases).where(eq(environmentLeases.id, id)))[0]!; + + it.each(["disk_full", "unknown_failure"] as const)("lets saved input continue after a file-preserving restart fails on a sandbox that cannot stop: %s", async failure => { + const f = await seed(), commentId = randomUUID(), queueId = randomUUID(); + const stopped = (await leaseRow(f.lease.id)).metadata?.remoteExecutionTermination; + f.runtime.resumeRunLease.mockRejectedValue(restartFailure(failure === "disk_full")); + f.runtime.retryPendingSandboxTeardown.mockRejectedValue(new Error("Sandbox is in an error state")); + const source = await run(f.runId); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: source.agentId, status: "running" }); + await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: f.issueId, + authorType: "user", authorUserId: "board", body: "Continue from a clean workspace." }); + await db.insert(agentWakeupRequests).values({ id: queueId, companyId: f.companyId, agentId: source.agentId, + source: "automation", reason: "issue_commented", status: "deferred_issue_execution", requestedByActorType: "user", requestedByActorId: "board", + payload: { issueId: f.issueId, commentId, _paperclipWakeContext: { issueId: f.issueId, wakeReason: "issue_commented", wakeCommentIds: [commentId], wakeCommentId: commentId } } }); + const attempts = failure === "disk_full" ? 1 : 6; + for (let attempt = 1; attempt <= attempts; attempt++) { + await recoverRetainedWorkspace(f.input); + if (attempt < attempts) await makeRetryDue(f.actionId); + } + await heartbeatService(db).resumeRemoteStopComments(await run(f.runId)); + const turns = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.status, "queued"))); + expect(turns).toHaveLength(1); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, queueId)))[0]).toMatchObject({ status: "coalesced", runId: turns[0].id }); + expect(await readFreshWorkspaceReadiness(db, turns[0])).toMatchObject({ mode: "fresh_isolated_workspace", + retryIntentId: queueId, sourceRunId: f.runId, retainedSourceIds: [f.lease.id] }); + const lease = await leaseRow(f.lease.id); + expect(lease).toMatchObject({ status: "released", cleanupStatus: "success" }); + expect(hasRemoteTerminationReceipt(lease)).toBe(true); + expect(lease.metadata?.remoteExecutionTermination).toEqual(stopped); + expect(lease.metadata).not.toHaveProperty("sandboxStopAndRetain"); + expect(readSystemRecovery((await action(f.actionId)).evidence)).toMatchObject({ state: "exhausted", attempts, + failure: failure === "disk_full" ? { code: "source_unavailable", retryClass: "use_alternative_strategy" } + : { code: "provider_unavailable", retryClass: "retry_transient" } }); + expect(f.runtime.resumeRunLease).toHaveBeenCalledTimes(attempts); + expect(await readFile(path.join(f.remote, "saved.txt"), "utf8")).toBe("finished work\n"); + }, 60_000); + + it("continues a Retry in a fresh workspace without repeating a restart that cannot succeed", async () => { + const f = await seed(); + f.runtime.resumeRunLease.mockRejectedValue(restartFailure(true)); + f.runtime.retryPendingSandboxTeardown.mockRejectedValue(new Error("Sandbox is in an error state")); + const source = await run(f.runId); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: source.agentId, status: "running" }); + await recoverRetainedWorkspace(f.input); + const exhausted = readSystemRecovery((await action(f.actionId)).evidence)!; + expect(exhausted).toMatchObject({ state: "exhausted", failure: { code: "source_unavailable", retryClass: "use_alternative_strategy" } }); + const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId: "local-board", + actorSource: "local_implicit", observedRunId: f.runId, idempotencyKey: randomUUID() }); + expect(request).toMatchObject({ accepted: true, stale: false }); + expect(readSystemRecovery((await action(f.actionId)).evidence)).toMatchObject({ generationId: exhausted.generationId, + state: "exhausted", attempts: 1, intentId: request.requestId }); + await recoverRetainedWorkspace(f.input); + expect(f.runtime.resumeRunLease).toHaveBeenCalledOnce(); + const heartbeat = heartbeatService(db); + await Promise.all([heartbeat.resumeTaskRetry(f.companyId, request.requestId!), heartbeat.resumeTaskRetry(f.companyId, request.requestId!)]); + const turns = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.status, "queued"))); + expect(turns).toHaveLength(1); + expect(turns[0].contextSnapshot).toMatchObject({ taskRetryRequestId: request.requestId, previousRunId: f.runId }); + expect(await readFreshWorkspaceReadiness(db, turns[0])).toMatchObject({ mode: "fresh_isolated_workspace", + retryIntentId: request.requestId, sourceRunId: f.runId }); + }, 30_000); + + it("keeps the stop obligation once the provider has confirmed a restart", async () => { + const f = await seed(); + f.runtime.resumeRunLease.mockResolvedValue({ providerLeaseId: f.providerLeaseId, metadata: { remoteCwd: path.join(f.remote, "elsewhere") } }); + f.runtime.retryPendingSandboxTeardown.mockRejectedValue(new Error("Provider stop unavailable")); + await recoverRetainedWorkspace(f.input); + const lease = await leaseRow(f.lease.id); + expect(lease).toMatchObject({ status: "pending_cleanup", cleanupStatus: "failed" }); + expect(hasRemoteTerminationReceipt(lease)).toBe(false); + expect(readStopOnlyCleanup(lease)).toMatchObject({ runId: f.runId, leaseId: f.lease.id }); + expect(lease.metadata?.sandboxStopAndRetain).not.toHaveProperty("priorStop"); + expect(f.runtime.retryPendingSandboxTeardown).toHaveBeenCalledOnce(); + }, 30_000); + + it("restores the stop receipt after an interrupted restart that the provider never confirmed", async () => { + const f = await seed(); + const stopped = (await leaseRow(f.lease.id)).metadata?.remoteExecutionTermination; + expect(await prepareStoppedSandboxForRecovery(db, await leaseRow(f.lease.id))).not.toBeNull(); + f.runtime.resumeRunLease.mockRejectedValue(restartFailure(true)); + f.runtime.retryPendingSandboxTeardown.mockRejectedValue(new Error("Sandbox is in an error state")); + await recoverRetainedWorkspace(f.input); + const lease = await leaseRow(f.lease.id); + expect(hasRemoteTerminationReceipt(lease)).toBe(true); + expect(lease.metadata?.remoteExecutionTermination).toEqual(stopped); + expect(readSystemRecovery((await action(f.actionId)).evidence)).toMatchObject({ state: "exhausted", + failure: { code: "source_unavailable", retryClass: "use_alternative_strategy" } }); + expect(f.runtime.retryPendingSandboxTeardown).toHaveBeenCalledTimes(2); + expect(f.runtime.resumeRunLease).toHaveBeenCalledOnce(); + }, 30_000); + it("discovers historical sources in the periodic sweep without creating a user request", async () => { const f = await seed("legacy"), onRecovered = vi.fn(async () => {}); await expect.poll(async () => { diff --git a/server/src/services/workspace-recovery.ts b/server/src/services/workspace-recovery.ts index cf0cc0f1de5..ea7e9b642b0 100644 --- a/server/src/services/workspace-recovery.ts +++ b/server/src/services/workspace-recovery.ts @@ -17,7 +17,7 @@ import { logger } from "../middleware/logger.js"; import type { EnvironmentRuntimeService } from "./environment-runtime.js"; import { resolveEnvironmentExecutionTarget } from "./environment-execution-target.js"; import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; -import { hasStopOnlyCleanup, prepareHistoricalSandboxStopVerification, prepareSandboxStopAndRetain, prepareStoppedSandboxForRecovery, settleAbsentSandbox, settleStopOnlyCleanup } from "./sandbox-stop-and-retain.js"; +import { confirmSandboxRecoveryRestart, hasStopOnlyCleanup, prepareHistoricalSandboxStopVerification, prepareSandboxStopAndRetain, prepareStoppedSandboxForRecovery, restoreSandboxStopAfterFailedRestart, settleAbsentSandbox, settleStopOnlyCleanup } from "./sandbox-stop-and-retain.js"; import { requestSandboxAbsenceProof } from "./sandbox-absence-proof.js"; import { persistActivity, publishActivity } from "./activity-log.js"; import { hasRequiredWorkspaceRecovery } from "./workspace-restore-recovery-state.js"; @@ -36,6 +36,7 @@ import { saveRemoteWorkspace } from "./remote-workspaces.js"; const exportCauses = ["native_workspace_sync_out_unrecoverable", "native_workspace_sync_out_retry_exhausted"]; const terminal = ["failed", "interrupted", "timed_out", "cancelled", "succeeded"]; +const sandboxDiskFull = /No space left on device|Disk quota exceeded/i; type Source = { companyId: string; issueId: string; runId: string; actionId: string }; type StopEvidenceSource = Omit & { actionId?: string }; @@ -241,12 +242,11 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ if (!pending) throw new SystemRecoveryFailure("agent_stop_unconfirmed", "recheck_evidence"); // The cleanup service owns exact-allocation termination. No process ID // or terminal run status is substituted for its provider receipt. - const receipt = await input.environmentRuntime.retryPendingSandboxTeardown({ - environment: environment as unknown as Environment, lease: pending as EnvironmentLease }).catch(async () => { - await settleStopOnlyCleanup(db, pending, { attemptId: ownership.token }); - throw new SystemRecoveryFailure("agent_stop_unconfirmed", "recheck_evidence"); - }); - const settled = await settleStopOnlyCleanup(db, pending, { attemptId: ownership.token, receipt }); + const settled = await input.environmentRuntime.retryPendingSandboxTeardown({ + environment: environment as unknown as Environment, lease: pending as EnvironmentLease }) + .then(receipt => settleStopOnlyCleanup(db, pending, { attemptId: ownership.token, receipt }), + async () => await restoreSandboxStopAfterFailedRestart(db, pending, { attemptId: ownership.token }) + ?? await settleStopOnlyCleanup(db, pending, { attemptId: ownership.token })); if (!settled || !hasRemoteTerminationReceipt(settled)) throw new SystemRecoveryFailure("agent_stop_unconfirmed", "recheck_evidence"); stoppedLease = settled; @@ -257,7 +257,7 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ if (target?.kind !== "remote" || target.transport !== "sandbox" || !target.runner || target.remoteCwd !== remoteCwd) throw workspaceSourceBindingFailure("execution_target_invalid"); await ownership.assertHeld(); - const cleanupLease = await db.transaction(async tx => { + let cleanupLease = await db.transaction(async tx => { const [fresh] = await tx.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.id, lease.id))).for("update"); if (!fresh || !hasRemoteTerminationReceipt(fresh) || fresh.providerLeaseId !== lease.providerLeaseId) @@ -278,7 +278,13 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ let stopSettled = false; try { const opened = await input.environmentRuntime.resumeWorkspaceRecoveryLease({ source: input, - leaseId: cleanupLease.id, environmentId: environment.id, ownership }); + leaseId: cleanupLease.id, environmentId: environment.id, ownership }).catch(error => { + throw sandboxDiskFull.test((error as Error | undefined)?.message ?? "") + ? new SystemRecoveryFailure("source_unavailable", "use_alternative_strategy") : error; + }); + const confirmed = await confirmSandboxRecoveryRestart(db, cleanupLease, { attemptId: ownership.token }); + if (!confirmed) throw new SystemRecoveryFailure("agent_stop_unconfirmed", "recheck_evidence"); + cleanupLease = confirmed; if (opened?.providerLeaseId !== lease.providerLeaseId) throw workspaceSourceBindingFailure("reopened_allocation_mismatch"); if (opened.metadata?.remoteCwd !== remoteCwd) throw workspaceSourceBindingFailure("reopened_cwd_mismatch"); await inspectWorkspaceRecoverySource(db, input); @@ -343,9 +349,10 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ // The maintenance worker must never turn it into destructive cleanup. try { await ownership.assertHeld(); - const receipt = await input.environmentRuntime.retryPendingSandboxTeardown({ environment: environment as unknown as Environment, - lease: cleanupLease as EnvironmentLease }); - await settleStopOnlyCleanup(db, cleanupLease, { attemptId: ownership.token, receipt }); + const settled = await input.environmentRuntime.retryPendingSandboxTeardown({ environment: environment as unknown as Environment, + lease: cleanupLease as EnvironmentLease }).then(receipt => settleStopOnlyCleanup(db, cleanupLease, { attemptId: ownership.token, receipt }), () => null); + if (!settled || !hasRemoteTerminationReceipt(settled)) + await restoreSandboxStopAfterFailedRestart(db, cleanupLease, { attemptId: ownership.token }); } catch { /* The persisted stop-only obligation remains authoritative. */ } } }