fix(recovery): preserve cancelled files after a reusable sandbox stops

A warm (keep_running) native run that is cancelled releases its reusable
sandbox through stop-and-retain. That release can finish before the
cancelled-workspace preservation worker sees a terminal coordinator. The
worker only accepted an active lease or a released ephemeral lease, so it
never recorded the file recovery. The task workspace kept the ended
writer's hold until the one-hour cutoff failed its successor runs.

Accept a released reusable lease with a confirmed stop-and-retain receipt
as a preservation source. The restore recorder still checks the receipt
and the exact workspace binding, and keeps the lease status and receipts.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

doc/execution-semantics.md

@@ -2429,6 +2429,16 @@ when Stop found no live controller. The worker retains the exact prepared source
 and obtains independent provider-stop evidence through normal recovery. It does
 not convert the no-op acknowledgement into a dispatched interrupt receipt or
 permission to continue the cancelled turn.
+Terminal release of a warm (`keep_running`) run that is not successful stops and
+retains its reusable sandbox. This release can finish before the preservation
+worker sees a terminal coordinator. A released reusable lease with a confirmed
+stop-and-retain receipt for the same run, lease and allocation therefore
+remains a preservation source. The worker records it as file recovery work and
+keeps its release status, policy and receipts. It dispatches no second stop and
+grants no execute right on the reusable sandbox. A released reusable lease without
+that receipt stays ineligible. Until this recovery saves the files, the
+task's remote workspace keeps the ended writer's hold. Successor runs wait for that
+recovery instead of overwriting unsaved files.
 A reusable lease can also owe native copyback: the export-repair action must
 select its exact immutable descriptor and the finalizer's accepted successful
 result under the same company, task, run and completion contract. Lease reuse

server/src/services/native-runtime/native-cancelled-workspace-preservation.test.ts

@@ -207,6 +207,91 @@ import { preserveCancelledNativeWorkspaces, preserveNativeFailedWorkspace,
     } finally { await worker.stop(); }
   });
 
+  /** A reassignment Stop of a warm (keep_running) run: the executor settles its
+   * coordinator, then terminal release stops and retains the reusable sandbox
+   * before the preservation worker's next check. */
+  async function stopAndRetainReusableSource(f: Awaited<ReturnType<typeof seed>>, options: { stopReceipt?: boolean } = {}) {
+    const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId));
+    const confirmedAt = new Date(lease.acquiredAt.getTime() + 1_000);
+    const releasedAt = new Date(confirmedAt.getTime() + 1);
+    const termination = { schema: "paperclip.remote-termination.v1", companyId: f.companyId, runId: f.runId,
+      leaseId: f.leaseId, provider: "daytona", providerLeaseId: f.reference.providerLeaseId, state: "stopped",
+      confirmedAt: confirmedAt.toISOString() };
+    const receipt = { schema: "paperclip.sandbox-stop-and-retain-receipt.v1", companyId: f.companyId, runId: f.runId,
+      leaseId: f.leaseId, provider: "daytona", providerLeaseId: f.reference.providerLeaseId, requestId: randomUUID(),
+      method: "environmentStopLease", pluginId: lease.metadata?.pluginId, confirmedAt: confirmedAt.toISOString() };
+    await db.update(environmentLeases).set({ status: "released", leasePolicy: "reuse_by_environment",
+      cleanupStatus: "success", releasedAt, metadata: { ...lease.metadata, pendingCleanupInFlight: false,
+        ...(options.stopReceipt === false ? {} : { remoteExecutionTermination: termination, sandboxStopAndRetainReceipt: receipt }) },
+    }).where(eq(environmentLeases.id, f.leaseId));
+    await db.update(executionWorkspaces).set({ mode: "shared_workspace" }).where(eq(executionWorkspaces.id, f.workspaceId));
+    await db.update(heartbeatRuns).set({ errorCode: "issue_reassigned", error: "Cancelled before issue reassignment",
+      runnerProfileJson: { remoteWorkspace: f.reference, nativeResourceDisposition: "keep_running" },
+    }).where(eq(heartbeatRuns.id, f.runId));
+    await db.update(nativeRunFinalizations).set({ failureCode: "native_retry_cancelled", recoveryState: null })
+      .where(eq(nativeRunFinalizations.runId, f.runId));
+    return (await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)))[0];
+  }
+
+  it.each(["worker", "startup-scan"] as const)("preserves files of a cancelled run whose reusable sandbox was already stopped and retained (%s)", async kind => {
+    const f = await seed();
+    const stopped = await stopAndRetainReusableSource(f);
+    const worker = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } });
+    try {
+      if (kind === "startup-scan") await preserveCancelledNativeWorkspaces(db);
+      else await registerCancelledWorkspacePreservationWork(db, worker, () => true, async () => true).ready;
+      const [retained] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
+      expect(retained.resultJson).toMatchObject({ nativeCancellation: f.nativeCancellation, workspaceRestoreFailure: "restore_failed",
+        workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.leaseId] } });
+      expect(retained).toMatchObject({ status: "cancelled", errorCode: "issue_reassigned", finishedAt: f.run.finishedAt });
+      // The stopped allocation keeps its receipts. Preservation dispatches no
+      // second stop and grants no execute right on the reusable sandbox.
+      expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)))[0]).toEqual(stopped);
+      const actions = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId));
+      expect(actions).toHaveLength(1);
+      expect(actions[0].evidence).toMatchObject({ runId: f.runId, workspaceRestoreRecovery: { leaseIds: [f.leaseId] } });
+      expect((await inspectWorkspaceRecoverySource(db, { companyId: f.companyId, issueId: f.issueId,
+        runId: f.runId, actionId: actions[0].id })).sources).toHaveLength(1);
+      if (kind === "worker") expect(worker.nextWakeAt()).toBeNull();
+      // A repeated pass records nothing again and starts no work.
+      await preserveCancelledNativeWorkspaces(db);
+      const repeated = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } });
+      try {
+        await registerCancelledWorkspacePreservationWork(db, repeated, () => true, async () => true).ready;
+        expect(repeated.nextWakeAt()).toBeNull();
+      } finally { await repeated.stop(); }
+      expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId))).toEqual(actions);
+      expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0].resultJson).toEqual(retained.resultJson);
+      expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1);
+      expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0);
+      expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0])
+        .toMatchObject({ phase: "terminal_failure", leaseOwner: null, resultId: null });
+    } finally { await worker.stop(); }
+  }, 120_000);
+
+  it.each(["live-owner", "pending-stop", "no-stop-receipt", "running"] as const)
+  ("does not preserve a stopped reusable sandbox for %s", async invalid => {
+    const f = await seed();
+    const stopped = await stopAndRetainReusableSource(f, { stopReceipt: invalid !== "no-stop-receipt" });
+    if (invalid === "live-owner") await db.update(nativeRunFinalizations).set({ phase: "observed", leaseOwner: randomUUID(),
+      leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.runId));
+    if (invalid === "pending-stop") await db.update(heartbeatRuns).set({ resultJson: {
+      nativeCancellation: { ...f.nativeCancellation, dispatchState: "pending" } } }).where(eq(heartbeatRuns.id, f.runId));
+    if (invalid === "running") await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, f.runId));
+    const errors: unknown[] = [];
+    const worker = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { errors.push(error); } });
+    try {
+      await registerCancelledWorkspacePreservationWork(db, worker, () => true, async () => true).ready;
+      await preserveCancelledNativeWorkspaces(db);
+      expect(errors).toEqual([]);
+      const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
+      expect(run.resultJson).not.toHaveProperty("workspaceRestoreRecovery");
+      expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)))[0]).toEqual(stopped);
+      expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.companyId, f.companyId))).toHaveLength(0);
+      expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0);
+    } finally { await worker.stop(); }
+  });
+
   it("rolls preservation back with its outer transaction", async () => {
     const f = await seed();
     await expect(db.transaction(async tx => {

server/src/services/native-runtime/native-failed-workspace-recovery.ts

@@ -16,7 +16,20 @@ import { logger } from "../../middleware/logger.js";
 import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
 import { preserveRemoteWorkspacePreparationFailure } from "./native-workspace-finalization-state.js";
 import { pinFailedObservedWorkspace } from "./native-observed-workspace-preservation.js";
+import { hasConfirmedSandboxStopAndRetain } from "../sandbox-stop-and-retain.js";
 
+/** Leases that can still hold a cancelled run's prepared files. Terminal
+ * release can stop and retain a reusable sandbox before preservation runs.
+ * Its confirmed stop receipt keeps those files as a recovery source; the
+ * restore recorder rechecks the receipt and the exact workspace binding. */
+function cancelledWorkspaceSourceLease() {
+  return or(eq(environmentLeases.status, "active"), and(
+    inArray(environmentLeases.status, ["pending_cleanup", "released"]), eq(environmentLeases.leasePolicy, "ephemeral"),
+  ), and(eq(environmentLeases.status, "released"),
+    inArray(environmentLeases.leasePolicy, ["reuse_by_environment", "reuse_by_execution_workspace"]),
+    sql`${environmentLeases.metadata}->'remoteExecutionTermination'->>'state' = 'stopped'`,
+    sql`${environmentLeases.metadata} ? 'sandboxStopAndRetainReceipt'`));
+}
 
 function workspaceRecoveryReference(profile: Record<string, unknown> | null | undefined) {
   const remote = remoteWorkspaceRunReferenceSchema.safeParse(profile?.remoteWorkspace);
@@ -178,9 +191,7 @@ export async function preserveCancelledNativeWorkspaces(db: Db) {
       eq(heartbeatRuns.id, environmentLeases.heartbeatRunId), eq(heartbeatRuns.companyId, environmentLeases.companyId),
     )).innerJoin(nativeRunFinalizations, and(eq(nativeRunFinalizations.runId, heartbeatRuns.id),
       eq(nativeRunFinalizations.companyId, heartbeatRuns.companyId), eq(nativeRunFinalizations.issueId, heartbeatRuns.nativeIssueId)))
-    .where(and(or(eq(environmentLeases.status, "active"), and(
+    .where(and(cancelledWorkspaceSourceLease(), eq(heartbeatRuns.runtimeMode, "native"),
-      inArray(environmentLeases.status, ["pending_cleanup", "released"]), eq(environmentLeases.leasePolicy, "ephemeral"),
-    )), eq(heartbeatRuns.runtimeMode, "native"),
       eq(heartbeatRuns.status, "cancelled"), eq(nativeRunFinalizations.phase, "terminal_failure"),
       isNull(nativeRunFinalizations.resultId), isNull(nativeRunFinalizations.leaseOwner),
       sql`(${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'state' = 'prepared' OR ${heartbeatRuns.runnerProfileJson}->'remoteWorkspace'->>'state' = 'prepared')`,
@@ -257,11 +268,12 @@ export function registerCancelledWorkspacePreservationWork(db: Db,
     if (finalization?.resultId) return null;
     if (finalization?.phase !== "terminal_failure" || finalization.leaseOwner)
       return { ...entry, dueAt: Date.now() + 30_000 };
-    const [source] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and(
+    const sources = await db.select().from(environmentLeases).where(and(
       eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),
-      or(eq(environmentLeases.status, "active"), and(inArray(environmentLeases.status, ["pending_cleanup", "released"]),
+      cancelledWorkspaceSourceLease(),
-        eq(environmentLeases.leasePolicy, "ephemeral"))),
+    ));
-    )).limit(1);
+    const source = sources.find(lease => lease.status !== "released" || lease.leasePolicy === "ephemeral"
+      || hasConfirmedSandboxStopAndRetain(lease));
     if (source) await preserveNativeFailedWorkspace(db, run, { allowPreterminal: true, requireCancelledStop: true });
     const [current] = await db.select().from(heartbeatRuns)
       .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId)));