diff --git a/doc/native-workspace-finalization-recovery.md b/doc/native-workspace-finalization-recovery.md index 7e0633e3f73..ad52162c20f 100644 --- a/doc/native-workspace-finalization-recovery.md +++ b/doc/native-workspace-finalization-recovery.md @@ -141,3 +141,18 @@ Daytona stop-only preservation disables provider auto-delete and refreshes the p New ephemeral native allocations receive an acquisition-time workspace sentinel bound to their run and provider allocation. Export-only resume requires that proof and the unchanged durable intent. A legacy ephemeral allocation without this proof remains blocked; recovery never creates a sentinel or accepts a replacement during resume. The reusable workspace identity hash is unchanged. Disabling auto-delete retains provider storage while repair waits. No automatic lease-age sweep deletes these released, confirmed-stopped allocations. Storage costs can continue until committed copyback applies the original cleanup policy or an operator explicitly deletes the allocation. + +### Generic stop-only cleanup + +An explicit sandbox `stop_and_retain` also requires the dedicated provider hook +when no export repair is involved, including successful per-turn reusable runs +and startup cancellation. Before dispatch, the controller stores a separate +`sandboxStopAndRetain` intent bound to the company, run, lease, provider allocation, +and original plugin. The initial cleanup and restart sweep both require that +plugin to advertise `environmentStopLease`. Missing capability or failed stop +leaves pending cleanup; ordinary release and destroy are never fallback methods. + +A matching stopped receipt clears the pending intent, leaves the lease released +and resumable, and saves `sandboxStopAndRetainReceipt`. Its request identity, +original plugin and `method: "environmentStopLease"` attest this dispatch route. +The accepted-result export intent and its commitment requirements are unchanged. diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 720845ccd55..199537cd055 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -483,14 +483,14 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { if (method === "environmentResumeLease") return { providerLeaseId: warmProviderLeaseId, metadata: { remoteCwd: "/workspace" }, }; - if (method === "environmentDestroyLease" || method === "environmentReleaseLease") return { + if (method === "environmentDestroyLease" || method === "environmentStopLease" || method === "environmentReleaseLease") return { providerLeaseId: input.providerLeaseId, state: method === "environmentDestroyLease" ? "destroyed" : "stopped", }; throw new Error(`Unexpected lifecycle method: ${method}`); }); const runtime = environmentRuntimeService(db, { pluginWorkerManager: { isRunning: () => true, call, - getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] }), + getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentStopLease", "environmentDestroyLease"] }), } as unknown as PluginWorkerManager }); const input = { companyId: seeded.companyId, agentId: seeded.agentId, issueId, @@ -531,11 +531,11 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { }); const workerManager = { isRunning: () => true, call, - getWorker: () => ({ supportedMethods: ["environmentReleaseLease"] }), + getWorker: () => ({ supportedMethods: ["environmentStopLease"] }), } as unknown as PluginWorkerManager; const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager }); await runtimeWithPlugin.releaseRunLeases(runId, "released", undefined, "stop_and_retain", true); - expect(call).toHaveBeenCalledWith(pluginId, "environmentReleaseLease", expect.objectContaining({ + expect(call).toHaveBeenCalledWith(pluginId, "environmentStopLease", expect.objectContaining({ companyId, providerLeaseId: reusableLease.providerLeaseId, cancelActiveWork: true, }), expect.any(Number)); expect(call).toHaveBeenCalledOnce(); @@ -544,6 +544,66 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { }, ); + it.each(["release_only", "stopped", "stop_failed", "unconfirmed", "restart", "competing_owner", "missing_pin", "late_receipt"])( + "never dispatches destructive release for an untagged stop-and-retain request: %s", async outcome => { + const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); + const lease = seeded.reusableLease; + await db.update(environmentLeases).set({ leasePolicy: "ephemeral", metadata: { + ...lease.metadata, reuseLease: false, ...(outcome === "missing_pin" ? { pluginId: undefined } : {}), + } }).where(eq(environmentLeases.id, lease.id)); + await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, seeded.runId)); + let supported = !["release_only", "restart"].includes(outcome); + let calls = 0; + const call = vi.fn(async (_id: string, method: string) => { + if (method !== "environmentStopLease") return { providerLeaseId: lease.providerLeaseId, state: "destroyed" }; + if (++calls === 1 && outcome === "stop_failed") throw new Error("injected stop failure"); + if (calls === 1 && outcome === "unconfirmed") return undefined; + if (outcome === "late_receipt") await db.update(environmentLeases).set({ + status: "active", heartbeatRunId: null, cleanupStatus: null, metadata: { newOwner: true }, + }).where(eq(environmentLeases.id, lease.id)); + return { providerLeaseId: lease.providerLeaseId, state: "stopped" }; + }); + const runtime = () => environmentRuntimeService(db, { pluginWorkerManager: { + isRunning: () => true, call, + getWorker: () => ({ supportedMethods: ["environmentReleaseLease", "environmentDestroyLease", ...(supported ? ["environmentStopLease"] : [])] }), + } as unknown as PluginWorkerManager }); + if (outcome === "competing_owner") await environmentService(db).acquireLease({ + companyId: seeded.companyId, environmentId: seeded.environment.id, heartbeatRunId: null, + leasePolicy: "ephemeral", provider: lease.provider, providerLeaseId: lease.providerLeaseId, + }); + await runtime().releaseRunLeases(seeded.runId, "failed", undefined, "stop_and_retain"); + expect(call.mock.calls.every(entry => entry[1] === "environmentStopLease")).toBe(true); + if (outcome === "late_receipt") { + expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "active", heartbeatRunId: null, metadata: { newOwner: true } }); + return; + } + if (outcome === "missing_pin") { + expect(call).not.toHaveBeenCalled(); + await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 }); + expect(call).not.toHaveBeenCalled(); + expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "pending_cleanup" }); + return; + } + if (["release_only", "restart", "competing_owner"].includes(outcome)) { + expect(call).not.toHaveBeenCalled(); + expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "pending_cleanup", metadata: { + sandboxStopAndRetain: { pluginId: seeded.pluginId, runId: seeded.runId, providerLeaseId: lease.providerLeaseId }, + } }); + await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 }); + expect(call).not.toHaveBeenCalled(); + if (outcome === "competing_owner") return; + supported = true; + } + if (["release_only", "stop_failed", "unconfirmed", "restart"].includes(outcome)) { + await heartbeatService(db, { environmentRuntime: runtime() }).sweepPendingCleanupLeases({ backoffMs: 0 }); + } + expect(await environmentService(db).getLeaseById(lease.id)).toMatchObject({ status: "released", cleanupStatus: "success", failureReason: null, + metadata: { remoteExecutionTermination: { state: "stopped", providerLeaseId: lease.providerLeaseId }, + sandboxStopAndRetainReceipt: { method: "environmentStopLease", runId: seeded.runId, pluginId: seeded.pluginId } } }); + expect(call.mock.calls.every(entry => entry[1] === "environmentStopLease")).toBe(true); + }, + ); + it("reports an unwired manager separately from a stopped sandbox worker", async () => { const { companyId, environment, runId } = await seedReusablePluginSandboxLease(); const input = { @@ -609,13 +669,14 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { const workerManager = { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { - if (method === "environmentReleaseLease") return undefined; + if (method === "environmentStopLease") return { providerLeaseId: reusableLease.providerLeaseId, state: "stopped" }; throw new Error(`Unexpected plugin method while stopping lease: ${method}`); }), getWorker: vi.fn(() => ({ supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", + "environmentStopLease", "environmentDestroyLease", ], })), @@ -635,7 +696,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { }); expect(workerManager.call).toHaveBeenCalledWith( pluginId, - "environmentReleaseLease", + "environmentStopLease", expect.objectContaining({ providerLeaseId: reusableLease.providerLeaseId }), expect.any(Number), ); @@ -663,12 +724,12 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { if (method === "environmentResumeLease") return { providerLeaseId: "projectless-1", metadata: { remoteCwd: "/workspace" }, }; - if (method === "environmentReleaseLease") return undefined; + if (method === "environmentStopLease") return { providerLeaseId: "projectless-1", state: "stopped" }; throw new Error(`Unexpected projectless lease method: ${method}`); }); const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: { isRunning: () => true, call, - getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] }), + getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentStopLease", "environmentDestroyLease"] }), } as unknown as PluginWorkerManager }); const first = await runtimeWithPlugin.acquireRunLease({ companyId: seeded.companyId, environment: seeded.environment, issueId, @@ -841,7 +902,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { }, }; } - if (method === "environmentReleaseLease") return undefined; + if (method === "environmentStopLease") return { providerLeaseId: "sandbox-exact-resume", state: "stopped" }; if (method === "environmentResumeLease") { return { providerLeaseId: "sandbox-exact-resume", @@ -860,6 +921,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { supportedMethods: [ "environmentResumeLease", "environmentReleaseLease", + "environmentStopLease", "environmentDestroyLease", ], })), diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index d168248bdad..c799eba7f2a 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -1,6 +1,7 @@ +import { hasStopOnlyCleanup, prepareSandboxStopAndRetain, readStopOnlyCleanup, settleStopOnlyCleanup, stopOnlyCleanupKey } from "./sandbox-stop-and-retain.js"; import { readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk"; import { remoteTerminationReceipt } from "./remote-execution-termination.js"; -import { hasNativeWorkspaceExportResume, readNativeWorkspaceExportResume, releaseCompletedNativeWorkspaceExportRetention, settleNativeWorkspaceExportResume } from "./native-runtime/native-workspace-export-resume.js"; +import { hasNativeWorkspaceExportResume, releaseCompletedNativeWorkspaceExportRetention } from "./native-runtime/native-workspace-export-resume.js"; import { createHash, randomUUID } from "node:crypto"; import { and, eq, inArray, ne, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -2573,8 +2574,8 @@ function createSandboxEnvironmentDriver( }, async retryPendingSandboxTeardown(input) { - const exportResume = hasNativeWorkspaceExportResume(input.lease); - const resumeIntent = readNativeWorkspaceExportResume(input.lease); + const exportResume = hasStopOnlyCleanup(input.lease); + const resumeIntent = readStopOnlyCleanup(input.lease); const assertExportResumeOwnership = async () => { if (!resumeIntent) throw new Error("Workspace export resume ownership is invalid."); const [current] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, input.lease.id)).limit(1); @@ -2584,8 +2585,8 @@ function createSandboxEnvironmentDriver( )).limit(1); if (!current || current.status !== "pending_cleanup" || current.companyId !== input.lease.companyId || current.metadata?.pendingCleanupAttemptId !== input.lease.metadata?.pendingCleanupAttemptId - || readNativeWorkspaceExportResume(current)?.requestId !== resumeIntent.requestId - || readNativeWorkspaceExportResume(current)?.pluginId !== resumeIntent.pluginId || otherOwner) { + || readStopOnlyCleanup(current)?.requestId !== resumeIntent.requestId + || readStopOnlyCleanup(current)?.pluginId !== resumeIntent.pluginId || otherOwner) { throw new Error("Workspace export resume ownership changed before cleanup."); } }; @@ -2648,12 +2649,12 @@ function createSandboxEnvironmentDriver( throw new Error("Workspace export recovery requires verified stop-only cleanup."); } await assertExportResumeOwnership(); - const receipt = await pluginWorkerManager.call(pluginId, "environmentStopLease", { + const receipt = await runLeaseReleaseWithRunParent(input.lease.id, () => pluginWorkerManager.call(pluginId, "environmentStopLease", { driverKey: recordedProvider, companyId: input.lease.companyId, environmentId: input.lease.environmentId ?? "", issueId: input.lease.issueId, config: workerConfig, providerLeaseId: input.lease.providerLeaseId, leaseMetadata: input.lease.metadata ?? {}, cancelActiveWork: true, resourceDisposition: "stop_and_retain", - }, Math.min(resolvePluginSandboxRpcTimeoutMs(workerConfig) ?? 60_000, 60_000)); + }, Math.min(resolvePluginSandboxRpcTimeoutMs(workerConfig) ?? 60_000, 60_000))); if (remoteTerminationReceipt(input.lease, receipt)?.state !== "stopped") { throw new Error("Workspace export recovery did not confirm the retained sandbox stopped."); } @@ -2722,7 +2723,7 @@ function createSandboxEnvironmentDriver( // teardown runs, throws its own "no worker manager" error, and counts // toward the cap. if (!pluginWorkerManager) return true; - if (hasNativeWorkspaceExportResume(input.lease)) { + if (hasStopOnlyCleanup(input.lease)) { const pinnedPluginId = readString(input.lease.metadata?.pluginId); if (!pinnedPluginId) return true; // Permanent invalid intent, never a by-key fallback. const pinned = await resolvePluginSandboxProviderDriverById({ db, pluginId: pinnedPluginId, driverKey: recordedProvider }); @@ -3215,6 +3216,9 @@ function createSandboxEnvironmentDriver( async function releasePluginBackedSandboxLease( input: EnvironmentDriverReleaseInput, ): Promise { + if (input.resourceDisposition === "stop_and_retain") { + throw new Error("Stop-only sandbox cleanup requires a durable stop intent."); + } const metadata = input.lease.metadata ?? {}; const pluginId = readString(metadata.pluginId); const providerKey = readString(metadata.provider); @@ -3243,7 +3247,6 @@ function createSandboxEnvironmentDriver( providerLeaseId: input.lease.providerLeaseId, leaseMetadata: metadata, ...(input.cancelActiveWork ? { cancelActiveWork: true } : {}), - ...(input.resourceDisposition ? { resourceDisposition: input.resourceDisposition } : {}), }, resolvePluginSandboxRpcTimeoutMs(stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig))), ); termination = remoteTerminationReceipt(input.lease, receipt); @@ -3928,7 +3931,7 @@ export function environmentRuntimeService( and( eq(environmentLeases.heartbeatRunId, heartbeatRunId), or(eq(environmentLeases.status, "active"), and(eq(environmentLeases.status, "pending_cleanup"), - sql`${environmentLeases.metadata} ? 'nativeWorkspaceExportResume'`)), + sql`(${environmentLeases.metadata} ? 'nativeWorkspaceExportResume' or ${environmentLeases.metadata} ? 'sandboxStopAndRetain')`)), ), ); if (leaseRows.length === 0) { @@ -3950,7 +3953,14 @@ export function environmentRuntimeService( const completed = await releaseCompletedNativeWorkspaceExportRetention(db, leaseSnapshot); if (completed) leaseSnapshot = toEnvironmentLeaseSnapshot(completed); } - if (hasNativeWorkspaceExportResume(leaseSnapshot)) { + if (providerResourceDisposition === "stop_and_retain" && !hasStopOnlyCleanup(leaseSnapshot) + && getLeaseDriverKey(leaseSnapshot, environment) === "sandbox") { + const prepared = await prepareSandboxStopAndRetain(db, leaseSnapshot); + if (!prepared) continue; + leaseSnapshot = toEnvironmentLeaseSnapshot(prepared); + } + if (hasStopOnlyCleanup(leaseSnapshot)) { + const intentKey = stopOnlyCleanupKey(leaseSnapshot); const attemptId = randomUUID(); const [claimed] = await db.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed", releasedAt: new Date(), @@ -3959,7 +3969,7 @@ export function environmentRuntimeService( }).where(and(eq(environmentLeases.id, leaseRow.id), eq(environmentLeases.heartbeatRunId, heartbeatRunId), inArray(environmentLeases.status, ["active", "pending_cleanup"]), sql`coalesce(${environmentLeases.metadata}->>'pendingCleanupInFlight', 'false') = 'false'`, - sql`${environmentLeases.metadata}->'nativeWorkspaceExportResume' = ${JSON.stringify(leaseRow.metadata?.nativeWorkspaceExportResume)}::jsonb`, + sql`${environmentLeases.metadata}->${intentKey} = ${JSON.stringify(leaseSnapshot.metadata?.[intentKey])}::jsonb`, )).returning(); if (!claimed) continue; const snapshot = toEnvironmentLeaseSnapshot(claimed); @@ -3967,11 +3977,11 @@ export function environmentRuntimeService( const driver = requireDriverKey(getLeaseDriverKey(snapshot, environment)); if (!driver.retryPendingSandboxTeardown) throw new Error("Workspace export recovery requires stop-only cleanup."); const receipt = await driver.retryPendingSandboxTeardown({ environment, lease: snapshot }); - const lease = await settleNativeWorkspaceExportResume(db, snapshot, { attemptId, receipt }); + const lease = await settleStopOnlyCleanup(db, snapshot, { attemptId, receipt }); if (lease && environment) released.push({ environment, lease: toEnvironmentLeaseSnapshot(lease), leaseContext: { executionWorkspaceId: lease.executionWorkspaceId, executionWorkspaceMode: null } }); } catch (error) { - await settleNativeWorkspaceExportResume(db, snapshot, { attemptId }); + await settleStopOnlyCleanup(db, snapshot, { attemptId }); throw error; } continue; diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 74cde9fa56c..9301d58e7c1 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -5,7 +5,7 @@ import { NativeWorkspaceFinalizationOwnershipLostError, type NativeWorkspaceFinalizationOwnership, } from "./native-runtime/native-workspace-finalization-ownership.js"; -import { hasNativeWorkspaceExportResume, settleNativeWorkspaceExportResume } from "./native-runtime/native-workspace-export-resume.js"; +import { hasStopOnlyCleanup, settleStopOnlyCleanup } from "./sandbox-stop-and-retain.js"; import { applyWorkspaceRestoreFailure } from "@paperclipai/adapter-utils/workspace-restore-result"; import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; import { externalConversationStateSql, nonIdleSlackIssueCondition } from "./slack-conversation-state.js"; @@ -18584,7 +18584,7 @@ export function heartbeatService( // An interrupted export resume also uses recorded cleanup, but its tagged // intent permits verified stop-and-retain only, never sandbox destruction. const isOrphanEphemeralLease = lease.leasePolicy === "ephemeral"; - const useRecordedTeardown = isOrphanEphemeralLease || !environment || hasNativeWorkspaceExportResume(lease); + const useRecordedTeardown = isOrphanEphemeralLease || !environment || hasStopOnlyCleanup(lease); // Do not consume a finite cleanup attempt while the provider plugin is // briefly unavailable. A plugin worker restart, a plugin reload, or a @@ -18653,8 +18653,8 @@ export function heartbeatService( environment, lease, }); - const released = hasNativeWorkspaceExportResume(lease) - ? await settleNativeWorkspaceExportResume(db, lease, { attemptId: claimed, receipt }) + const released = hasStopOnlyCleanup(lease) + ? await settleStopOnlyCleanup(db, lease, { attemptId: claimed, receipt }) : await environmentsSvc.releaseLease(lease.id, "expired", { expectedPendingCleanupAttemptId: claimed, cleanupStatus: "success", diff --git a/server/src/services/sandbox-stop-and-retain.ts b/server/src/services/sandbox-stop-and-retain.ts new file mode 100644 index 00000000000..ce4e54acdea --- /dev/null +++ b/server/src/services/sandbox-stop-and-retain.ts @@ -0,0 +1,85 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, sql, type SQL } from "drizzle-orm"; +import { environmentLeases, type Db } from "@paperclipai/db"; +import { remoteTerminationReceipt } from "./remote-execution-termination.js"; +import { hasNativeWorkspaceExportResume, readNativeWorkspaceExportResume, settleNativeWorkspaceExportResume } from "./native-runtime/native-workspace-export-resume.js"; + +type Lease = Pick; +export const SANDBOX_STOP_AND_RETAIN_KEY = "sandboxStopAndRetain"; + +export function hasStopOnlyCleanup(lease: Pick): boolean { + return hasNativeWorkspaceExportResume(lease) + || Object.prototype.hasOwnProperty.call(lease.metadata ?? {}, SANDBOX_STOP_AND_RETAIN_KEY); +} + +export function stopOnlyCleanupKey(lease: Pick) { + return hasNativeWorkspaceExportResume(lease) ? "nativeWorkspaceExportResume" : SANDBOX_STOP_AND_RETAIN_KEY; +} + +export function readStopOnlyCleanup(lease: Lease) { + if (hasNativeWorkspaceExportResume(lease)) return readNativeWorkspaceExportResume(lease); + const value = lease.metadata?.[SANDBOX_STOP_AND_RETAIN_KEY]; + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const intent = value as Record; + if (intent.schema !== "paperclip.sandbox-stop-and-retain.v1" + || intent.companyId !== lease.companyId || intent.runId !== lease.heartbeatRunId || !lease.heartbeatRunId + || intent.leaseId !== lease.id || intent.provider !== lease.provider || !lease.provider + || intent.providerLeaseId !== lease.providerLeaseId || !lease.providerLeaseId + || typeof intent.requestId !== "string" || !intent.requestId + || typeof intent.pluginId !== "string" || !intent.pluginId || intent.pluginId !== lease.metadata?.pluginId) return null; + return { ...intent, requestId: intent.requestId, pluginId: intent.pluginId }; +} + +/** Persist before provider dispatch so an older worker or a restart cannot turn + * an explicit stop into ordinary destructive release. Invalid pins stay pending. */ +export async function prepareSandboxStopAndRetain(db: Db, lease: Lease) { + const requestId = randomUUID(), now = new Date(); + const intent = { schema: "paperclip.sandbox-stop-and-retain.v1", requestId, + companyId: lease.companyId, runId: lease.heartbeatRunId, leaseId: lease.id, + provider: lease.provider, providerLeaseId: lease.providerLeaseId, pluginId: lease.metadata?.pluginId }; + const [updated] = await db.update(environmentLeases).set({ + status: "pending_cleanup", cleanupStatus: "failed", failureReason: "sandbox_stop_pending", releasedAt: now, updatedAt: now, + metadata: sql`(coalesce(${environmentLeases.metadata}, '{}'::jsonb) - 'remoteExecutionTermination' - 'sandboxStopAndRetainReceipt') || ${JSON.stringify({ + [SANDBOX_STOP_AND_RETAIN_KEY]: intent, pendingCleanupAttemptId: requestId, + pendingCleanupInFlight: false, pendingCleanupLeaseExpiresAtMs: 0, + pendingCleanupRetryAfterMs: 0, pendingCleanupRetryAttempts: 0, pendingCleanupRetryCapWarned: false, + })}::jsonb`, + }).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId), + eq(environmentLeases.heartbeatRunId, lease.heartbeatRunId!), eq(environmentLeases.status, "active"), + sql`${environmentLeases.provider} is not distinct from ${lease.provider}`, + sql`${environmentLeases.providerLeaseId} is not distinct from ${lease.providerLeaseId}`, + sql`${environmentLeases.metadata} is not distinct from ${lease.metadata === null ? null : JSON.stringify(lease.metadata)}::jsonb`, + )).returning(); + return updated ?? null; +} + +export async function settleStopOnlyCleanup(db: Db, lease: Lease, options: { attemptId: string; receipt?: unknown }) { + if (hasNativeWorkspaceExportResume(lease)) return settleNativeWorkspaceExportResume(db, lease, options); + const intent = readStopOnlyCleanup(lease); + if (!intent) return null; + const receipt = remoteTerminationReceipt(lease, options.receipt), stopped = receipt?.state === "stopped"; + const now = new Date(); + const removeIntent: SQL = stopped ? sql`- 'sandboxStopAndRetain'` : sql``; + const [updated] = await db.update(environmentLeases).set({ + status: stopped ? "released" : "pending_cleanup", cleanupStatus: stopped ? "success" : "failed", + failureReason: stopped ? null : "sandbox_stop_pending", + releasedAt: now, lastUsedAt: now, updatedAt: now, + metadata: sql`(${environmentLeases.metadata} - 'remoteExecutionTermination' ${removeIntent}) || ${JSON.stringify({ + ...(stopped ? { remoteExecutionTermination: receipt, sandboxStopAndRetainReceipt: { + schema: "paperclip.sandbox-stop-and-retain-receipt.v1", requestId: intent.requestId, + companyId: lease.companyId, runId: lease.heartbeatRunId, leaseId: lease.id, + provider: lease.provider, providerLeaseId: lease.providerLeaseId, pluginId: intent.pluginId, + method: "environmentStopLease", confirmedAt: receipt.confirmedAt, + } } : {}), + pendingCleanupInFlight: false, pendingCleanupLeaseExpiresAtMs: 0, + })}::jsonb`, + }).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId), + eq(environmentLeases.heartbeatRunId, lease.heartbeatRunId!), eq(environmentLeases.provider, lease.provider!), + eq(environmentLeases.providerLeaseId, lease.providerLeaseId!), eq(environmentLeases.status, "pending_cleanup"), + sql`${environmentLeases.metadata}->>'pendingCleanupAttemptId' = ${options.attemptId}`, + sql`${environmentLeases.metadata}->'sandboxStopAndRetain'->>'requestId' = ${intent.requestId}`, + sql`${environmentLeases.metadata}->>'pluginId' = ${intent.pluginId}`, + )).returning(); + return updated ?? null; +}