Recognize confirmed container loss when resuming Daytona leases

Daytona can retain a sandbox API record after its container disappears.
Confirm the exact missing-container response with a bounded fresh read,
then let the host apply its existing replacement and backup policy.
Preserve unknown failures, recoverable errors, and identity mismatches.

Verified 251 provider tests, 93 host lifecycle tests, plugin typecheck and
build. Added 15 regressions. Read-only inspection confirmed the provider
response and an existing verified backup without changing live state.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/plugins/sandbox-providers/daytona/README.md

@@ -28,6 +28,7 @@ Notes:
 - The current published Daytona SDK package is `@daytonaio/sdk`.
 - The driver supports both `snapshot`-based and `image`-based sandbox creation. If both are set, validation rejects the config as ambiguous.
 - Reusable leases map to Daytona stop/start semantics. Non-reusable leases are deleted on release. A provider-resolved `target` does not change the identity of an existing sandbox. Release closes the same scoped lease that a later sentinel-verified resume reopens.
+- A sandbox record can survive the loss of its underlying container. Resume treats it as expired only when a fresh provider read confirms the exact missing-container error for that sandbox and marks it unrecoverable. Unknown errors and failed confirmation reads preserve the lease. The host still requires a verified native-runner backup before replacement.
 
 ## Local development
 

packages/plugins/sandbox-providers/daytona/src/plugin.test.ts

@@ -1264,6 +1264,132 @@ describe("Daytona sandbox provider plugin", () => {
     });
   });
 
+  describe("missing-container resume", () => {
+    const sandboxId = "00000000-0000-4000-8000-000000000001";
+    const missing = `not found: failed to inspect sandbox container ${sandboxId}: Error response from daemon: No such container: ${sandboxId}`;
+    const params = {
+      driverKey: "daytona", companyId: "company-1", environmentId: "env-1", providerLeaseId: sandboxId,
+      config: { apiKey: "host-key", timeoutMs: 300000, livenessTimeoutMs: 100, reuseLease: true },
+      leaseMetadata: { workspaceSentinel: { path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json", token: "sentinel-token" } },
+    };
+    const resume = () => plugin.definition.onEnvironmentResumeLease!(params);
+    function missingSandbox() {
+      return { ...createMockSandbox({ id: sandboxId, state: "error", recoverable: false }), errorReason: missing };
+    }
+    function allowSentinel(sandbox: ReturnType<typeof missingSandbox>) {
+      sandbox.process.executeCommand.mockResolvedValueOnce({ exitCode: 0,
+        result: JSON.stringify({ token: "sentinel-token" }), artifacts: { stdout: JSON.stringify({ token: "sentinel-token" }) },
+      });
+    }
+
+    it("expires a provider record with a freshly confirmed missing container without replacing it", async () => {
+      const sandbox = missingSandbox();
+      mockGet.mockResolvedValue(sandbox);
+      for (let attempt = 0; attempt < 2; attempt++) {
+        await expect(resume()).resolves.toEqual({ providerLeaseId: null, metadata: { expired: true } });
+      }
+      expect(mockGet).toHaveBeenCalledTimes(2);
+      expect(sandbox.refreshData).toHaveBeenCalledTimes(2);
+      expect(sandbox.start).not.toHaveBeenCalled();
+      expect(sandbox.recover).not.toHaveBeenCalled();
+      expect(sandbox.delete).not.toHaveBeenCalled();
+      expect(mockCreate).not.toHaveBeenCalled();
+    });
+
+    it.each(["sandbox-opaque", "sandbox.with+[literal](characters)"])("matches the opaque sandbox ID literally: %s", async (id) => {
+      const sandbox = { ...missingSandbox(), id, errorReason: missing.replaceAll(sandboxId, id) };
+      mockGet.mockResolvedValue(sandbox);
+      await expect(plugin.definition.onEnvironmentResumeLease!({ ...params, providerLeaseId: id }))
+        .resolves.toEqual({ providerLeaseId: null, metadata: { expired: true } });
+      expect(sandbox.refreshData).toHaveBeenCalledOnce();
+      expect(sandbox.delete).not.toHaveBeenCalled();
+      expect(mockCreate).not.toHaveBeenCalled();
+    });
+
+    it.each([
+      "not found: provider temporarily unavailable",
+      missing.replace("No such container:", "No such volume:"),
+      missing.replace(/000000000001$/, "000000000002"),
+      missing.replaceAll(sandboxId, "00000000-0000-4000-8000-000000000002"),
+    ])("preserves an unexplained unrecoverable error: %s", async (errorReason) => {
+      const sandbox = { ...missingSandbox(), errorReason };
+      mockGet.mockResolvedValue(sandbox);
+      await expect(resume()).rejects.toThrow("unrecoverable error state");
+      expect(sandbox.delete).not.toHaveBeenCalled();
+      expect(mockCreate).not.toHaveBeenCalled();
+    });
+
+    it("uses provider recovery when the sandbox is recoverable", async () => {
+      const sandbox = { ...missingSandbox(), recoverable: true };
+      allowSentinel(sandbox);
+      mockGet.mockResolvedValue(sandbox);
+      await expect(resume()).resolves.toMatchObject({ providerLeaseId: sandboxId });
+      expect(sandbox.recover).toHaveBeenCalled();
+      expect(sandbox.delete).not.toHaveBeenCalled();
+    });
+
+    it("reuses a sandbox whose fresh state disproves the cached loss", async () => {
+      const sandbox = missingSandbox();
+      sandbox.refreshData.mockImplementation(async () => { sandbox.state = "started"; });
+      allowSentinel(sandbox);
+      mockGet.mockResolvedValue(sandbox);
+      await expect(resume()).resolves.toMatchObject({ providerLeaseId: sandboxId });
+      expect(sandbox.start).not.toHaveBeenCalled();
+      expect(sandbox.delete).not.toHaveBeenCalled();
+      expect(mockCreate).not.toHaveBeenCalled();
+    });
+
+    it.each([new MockDaytonaTimeoutError("timed out"), new Error("provider 503")])("preserves the lease if confirmation fails: %s", async (error) => {
+      const sandbox = missingSandbox();
+      sandbox.refreshData.mockRejectedValue(error);
+      mockGet.mockResolvedValue(sandbox);
+      await expect(resume()).rejects.toThrow(error.message);
+      expect(sandbox.delete).not.toHaveBeenCalled();
+      expect(mockCreate).not.toHaveBeenCalled();
+    });
+
+    it("preserves an unknown error returned by the fresh provider read", async () => {
+      const sandbox = missingSandbox();
+      sandbox.refreshData.mockImplementation(async () => { sandbox.errorReason = "provider unavailable"; });
+      mockGet.mockResolvedValue(sandbox);
+      await expect(resume()).rejects.toThrow("unrecoverable error state");
+      expect(sandbox.delete).not.toHaveBeenCalled();
+      expect(mockCreate).not.toHaveBeenCalled();
+    });
+
+    it("rejects a refreshed handle belonging to a different sandbox", async () => {
+      const sandbox = missingSandbox();
+      sandbox.refreshData.mockImplementation(async () => { sandbox.id = "another-sandbox"; });
+      mockGet.mockResolvedValue(sandbox);
+      await expect(resume()).rejects.toThrow("handle mismatch");
+      expect(sandbox.delete).not.toHaveBeenCalled();
+      expect(mockCreate).not.toHaveBeenCalled();
+    });
+
+    it("accepts a typed not-found result while confirming the missing container", async () => {
+      const sandbox = missingSandbox();
+      sandbox.refreshData.mockRejectedValue(new MockDaytonaNotFoundError("missing"));
+      mockGet.mockResolvedValue(sandbox);
+      await expect(resume()).resolves.toEqual({ providerLeaseId: null, metadata: { expired: true } });
+      expect(sandbox.delete).not.toHaveBeenCalled();
+      expect(mockCreate).not.toHaveBeenCalled();
+    });
+
+    it("bounds a stalled confirmation and preserves the lease", async () => {
+      vi.useFakeTimers();
+      try {
+        const sandbox = missingSandbox();
+        sandbox.refreshData.mockImplementation(() => new Promise(() => {}));
+        mockGet.mockResolvedValue(sandbox);
+        const result = resume().then(() => null, error => error as Error);
+        await vi.advanceTimersByTimeAsync(101);
+        expect((await result)?.message).toContain("sandbox.refreshData");
+        expect(sandbox.delete).not.toHaveBeenCalled();
+        expect(mockCreate).not.toHaveBeenCalled();
+      } finally { vi.useRealTimers(); }
+    });
+  });
+
   it("expires a reusable lease when the workspace sentinel does not match", async () => {
     process.env.DAYTONA_API_KEY = "host-key";
     const sandbox = createMockSandbox({ id: "sandbox-reuse", state: "stopped" });

packages/plugins/sandbox-providers/daytona/src/plugin.ts

@@ -502,6 +502,13 @@ async function ensureSandboxStarted(sandbox: Sandbox, timeoutSeconds: number): P
   await withLivenessTimeout("sandbox.start", startBoundMs, () => sandbox.start(timeoutSeconds));
 }
 
+function hasMissingSandboxContainer(sandbox: Sandbox): boolean {
+  if (sandbox.state !== "error" || sandbox.recoverable !== false || typeof sandbox.errorReason !== "string") return false;
+  if (typeof sandbox.id !== "string" || !sandbox.id) return false;
+  return sandbox.errorReason ===
+    `not found: failed to inspect sandbox container ${sandbox.id}: Error response from daemon: No such container: ${sandbox.id}`;
+}
+
 async function resolveSandboxWorkingDirectory(sandbox: Sandbox): Promise<string> {
   const root = (await sandbox.getWorkDir())?.trim()
     || (await sandbox.getUserHomeDir())?.trim()
@@ -2228,6 +2235,26 @@ const plugin = definePlugin({
         return { providerLeaseId: null, metadata: { expired: true } };
       }
 
+      // Daytona can retain the API record after losing its container. Confirm
+      // the exact provider report again before allowing the host's existing
+      // backup-guarded replacement path. Unknown errors preserve the lease.
+      if (hasMissingSandboxContainer(sandbox)) {
+        try {
+          await withLivenessTimeout("sandbox.refreshData", config.livenessTimeoutMs, () => sandbox.refreshData());
+          assertHandleMatchesLease(sandbox, params.providerLeaseId);
+        } catch (error) {
+          evictSandboxHandle(scope);
+          if (error instanceof DaytonaNotFoundError) {
+            return { providerLeaseId: null, metadata: { expired: true } };
+          }
+          throw error;
+        }
+        if (hasMissingSandboxContainer(sandbox)) {
+          evictSandboxHandle(scope);
+          return { providerLeaseId: null, metadata: { expired: true } };
+        }
+      }
+
         // A stopped sandbox loses its session shell, so the stored session id is
         // stale after a real restart. Clear the id only when the sandbox is not
         // already running, and clear it before the restart. A stopped sandbox has