Preserve reusable sandboxes through failed resume and stop

server/src/__tests__/environment-runtime.test.ts

@@ -470,6 +470,48 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
     return { pluginId, companyId, agentId, environment, runId, executionWorkspaceId, reusableLease };
   }
 
+  it.each(["codex_local", "paperclip_runner"])("retains a failed %s resume through run cleanup and retries the original sandbox", async (adapterType) => {
+    const seeded = await seedReusablePluginSandboxLease(adapterType);
+    let failResume = false;
+    const workerManager = {
+      isRunning: vi.fn(() => true),
+      getWorker: vi.fn(() => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] })),
+      call: vi.fn(async (_pluginId: string, method: string) => {
+        if (method !== "environmentResumeLease") throw new Error(`Unexpected disposal or replacement: ${method}`);
+        if (failResume) throw new Error("Sandbox is not in a startable state");
+        return { providerLeaseId: seeded.reusableLease.providerLeaseId, metadata: {
+          provider: "fake-plugin", image: "fake:test", timeoutMs: 1234, reuseLease: true,
+        } };
+      }),
+    } as unknown as PluginWorkerManager;
+    const runtime = environmentRuntimeService(db, { pluginWorkerManager: workerManager });
+    const acquire = (runId: string) => runtime.acquireRunLease({
+      companyId: seeded.companyId, environment: seeded.environment, agentId: seeded.agentId,
+      heartbeatRunId: runId, issueId: null, adapterType,
+      persistedExecutionWorkspace: { id: seeded.executionWorkspaceId, mode: "shared_workspace" },
+    });
+    const first = await acquire(seeded.runId);
+    await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, seeded.runId));
+    await environmentService(db).releaseLease(first.lease.id, "released");
+    const failedRun = randomUUID();
+    await db.insert(heartbeatRuns).values({ id: failedRun, companyId: seeded.companyId, agentId: seeded.agentId, status: "running" });
+    failResume = true;
+    await expect(acquire(failedRun)).rejects.toThrow("the lease was preserved");
+    await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, failedRun));
+    const released = await runtime.releaseRunLeases(failedRun, "failed");
+    expect(released).toHaveLength(1);
+    expect(released[0]!.lease).toMatchObject({ status: "retained", expiresAt: null,
+      providerLeaseId: first.lease.providerLeaseId, metadata: { sandboxResumePending: true } });
+    expect(vi.mocked(workerManager.call).mock.calls.every((call) => call[1] === "environmentResumeLease")).toBe(true);
+    const retryRun = randomUUID();
+    await db.insert(heartbeatRuns).values({ id: retryRun, companyId: seeded.companyId, agentId: seeded.agentId, status: "running" });
+    failResume = false;
+    const retry = await acquire(retryRun);
+    expect(retry.lease.providerLeaseId).toBe(first.lease.providerLeaseId);
+    expect(retry.lease.metadata?.sandboxResumePending).toBe(false);
+    expect(retry.lease.metadata?.sandboxLeaseAcquisition).toEqual({ outcome: "resumed" });
+  });
+
   it("keeps an existing task's legacy sync contract after its sandbox expires, without affecting new tasks", async () => {
     const seeded = await seedReusablePluginSandboxLease("codex_local");
     const taskId = randomUUID();
@@ -5405,7 +5447,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
     return { pluginId, environment, runId, lease, workerManager, runtimeWithPlugin };
   }
 
-  it("routes release to pending_cleanup when the worker no longer advertises the release lifecycle method", async () => {
+  it("retains reusable work when the worker no longer advertises the release lifecycle method", async () => {
     const { pluginId, lease, workerManager, runtimeWithPlugin } = await seedStaleLifecycleReusableLease();
 
     const released = await runtimeWithPlugin.releaseRunLeases(lease.heartbeatRunId!);
@@ -5417,11 +5459,9 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
       expect.anything(),
       expect.anything(),
     );
-    // The failed release verification must enter the pending-cleanup retry flow.
-    // The reaper sweeps only `pending_cleanup` leases, so a `released` status
-    // here would strand the still-active provider resource.
+    // The destructive orphan sweep must not discard reusable work.
     await expect(environmentService(db).getLeaseById(lease.id)).resolves.toMatchObject({
-      status: "pending_cleanup",
+      status: "retained",
       cleanupStatus: "failed",
       failureReason: "release_cleanup_failed",
     });

server/src/services/environment-runtime.ts

@@ -696,6 +696,27 @@ export class SandboxOrphanCleanupWriteError extends Error {
   }
 }
 
+async function retainIncompleteSandboxResume(db: Db, lease: EnvironmentLease): Promise<EnvironmentLease | null> {
+  if (lease.metadata?.sandboxResumePending !== true || !lease.heartbeatRunId) return null;
+  // A resume failure is not proof that the existing workspace is disposable.
+  // Keep the claimed reference eligible for retry, without a provider release
+  // RPC or the destructive pending-cleanup sweep. Successful acquisition
+  // replaces this provisional metadata with the verified provider metadata.
+  const now = new Date();
+  const [retained] = await db.update(environmentLeases).set({
+    status: "retained", expiresAt: null, releasedAt: null,
+    failureReason: "sandbox_resume_incomplete", cleanupStatus: "failed",
+    lastUsedAt: now, updatedAt: now,
+  }).where(and(
+    eq(environmentLeases.id, lease.id),
+    eq(environmentLeases.companyId, lease.companyId),
+    eq(environmentLeases.heartbeatRunId, lease.heartbeatRunId),
+    inArray(environmentLeases.status, ["active", "retained"]),
+    sql`${environmentLeases.metadata}->'sandboxResumePending' = 'true'::jsonb`,
+  )).returning();
+  return retained ? toEnvironmentLeaseSnapshot(retained) : null;
+}
+
 /** A reusable sandbox could not be resumed, but has not been proven lost. */
 export class ReusableSandboxResumeError extends Error {
   readonly provider: string;
@@ -1299,7 +1320,13 @@ function createSandboxEnvironmentDriver(
     lease: EnvironmentLease,
     input: Parameters<EnvironmentRuntimeDriver["acquireRunLease"]>[0],
   ): Promise<EnvironmentLease> {
-    if (!input.heartbeatRunId || lease.heartbeatRunId === input.heartbeatRunId) return lease;
+    if (!input.heartbeatRunId) return lease;
+    const metadata = { ...lease.metadata, sandboxResumePending: true };
+    if (lease.heartbeatRunId === input.heartbeatRunId) {
+      const claimed = await environmentsSvc.updateLeaseMetadata(lease.id, metadata);
+      if (!claimed) throw new Error("Reusable sandbox claim disappeared before resume");
+      return claimed;
+    }
     // Transfer ownership atomically before touching the provider. Two server
     // processes can observe the same released lease; only the winner of this
     // conditional update/insert may resume its sandbox. A failed resume leaves
@@ -1317,7 +1344,7 @@ function createSandboxEnvironmentDriver(
       // The retained provider's old expiry may already be past. The resume RPC
       // will attest its current expiry; until then use only this run's deadline.
       expiresAt: input.requestedExpiresAt ?? null,
-      metadata: lease.metadata,
+      metadata,
       replacesReusableLeaseId: lease.id,
     });
   }
@@ -2156,6 +2183,7 @@ function createSandboxEnvironmentDriver(
           sandboxProviderPlugin: true,
           ...sandboxConfigForLeaseMetadata(storedConfig),
           ...sanitizedProviderMetadata,
+          sandboxResumePending: false,
           workFolderLayout: legacyWorkFolderLayout || (reusableLease && hasLegacySandboxWorkspace(reusableLease)) ? "legacy" : "scoped",
           sandboxLeaseAcquisition: providerLease
             ? {
@@ -2407,6 +2435,7 @@ function createSandboxEnvironmentDriver(
         driver: input.environment.driver,
         executionWorkspaceMode: input.executionWorkspaceMode,
         ...providerLease.metadata,
+        sandboxResumePending: false,
         workFolderLayout: legacyWorkFolderLayout || (reusableLease && hasLegacySandboxWorkspace(reusableLease)) ? "legacy" : "scoped",
         sandboxLeaseAcquisition:
           reusableLease && providerLease.providerLeaseId === reusableLease.providerLeaseId
@@ -2502,6 +2531,8 @@ function createSandboxEnvironmentDriver(
     },
 
     async releaseRunLease(input) {
+      const pendingResume = await retainIncompleteSandboxResume(db, input.lease);
+      if (pendingResume) return pendingResume;
       if (await retainUnsavedWorkFolderLease(db, input.lease)) return { ...input.lease, status: "retained", expiresAt: null, failureReason: "work_folder_save_required" };
       if (input.status === "expired" && input.lease.leasePolicy === "reuse_by_environment") {
         return await destroyReusableSandboxLease({
@@ -2547,7 +2578,7 @@ function createSandboxEnvironmentDriver(
       } catch {
         cleanupStatus = "failed";
       }
-      const releaseStatus = input.lease.leasePolicy === "retain_on_failure" && input.status === "failed"
+      const releaseStatus = (input.lease.leasePolicy === "retain_on_failure" || input.lease.leasePolicy === "reuse_by_environment") && input.status === "failed"
         ? "retained" as const
         : input.status;
       return await environmentsSvc.releaseLease(input.lease.id, releaseStatus, {
@@ -3155,13 +3186,12 @@ function createSandboxEnvironmentDriver(
       cleanupStatus = "failed";
     }
 
-    // A failed release verification leaves the provider resource active. The
-    // cleanup reaper retries only `pending_cleanup` leases, so route a failed
-    // release into that retry flow. A `retain_on_failure` lease keeps the
-    // resource on purpose for reuse, so it stays `retained` and never enters the
-    // reaper, which would destroy the resource the retain policy wants to keep.
+    // Reusable work must survive a failed stop. Never send that resource to the
+    // destructive pending-cleanup sweep; the next exact-lease resume can retry.
+    // Ephemeral orphan cleanup still uses the existing sweep.
     const retained =
-      input.lease.leasePolicy === "retain_on_failure" && input.status === "failed";
+      (input.lease.leasePolicy === "retain_on_failure" && input.status === "failed") ||
+      (input.lease.leasePolicy === "reuse_by_environment" && (input.status === "failed" || cleanupStatus === "failed"));
     const releaseStatus = retained
       ? ("retained" as const)
       : cleanupStatus === "failed"
@@ -3318,6 +3348,7 @@ const INTERNAL_PLUGIN_SANDBOX_CONFIG_KEYS = new Set([
   "shellCommand",
   "sandboxProviderPlugin",
   "sandboxLeaseAcquisition",
+  "sandboxResumePending",
   "nativeHarnessBackup",
   "nativeWorkspaceSync",
 ]);
@@ -3837,6 +3868,19 @@ export function environmentRuntimeService(
           if (!environment) continue;
 
           const leaseSnapshot = toEnvironmentLeaseSnapshot(leaseRow);
+          const pendingResume = await retainIncompleteSandboxResume(db, leaseSnapshot);
+          if (pendingResume) {
+            released.push({
+              environment,
+              lease: pendingResume,
+              leaseContext: {
+                executionWorkspaceId: pendingResume.executionWorkspaceId,
+                executionWorkspaceMode:
+                  (pendingResume.metadata?.executionWorkspaceMode as ExecutionWorkspace["mode"] | null | undefined) ?? null,
+              },
+            });
+            continue;
+          }
           if (await retainUnsavedWorkFolderLease(db, leaseSnapshot)) continue;
           if (
             providerResourceDisposition === "keep_running" &&