fix(server): recover remote agent files on an empty controller

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/DEVELOPING.md

@@ -4181,7 +4181,15 @@ Runner and remote workspace storage. A fresh run still has its initial `legacy`
 label at this phase; that label is not the selected execution mode. The runtime
 selector checks the resolved native warm lifecycle before admission and rejects
 legacy or per-turn execution. External conversation providers keep their existing
-exemption because they own no task filesystem.
+exemption because they own no task filesystem. Source resolution checks this
+contract again before a configured host folder can fall back to local storage.
+
+A replacement controller reconstructs an active remote agent-directory cache
+from the registered sandbox, including unsaved files. The remote directory,
+lease and save baseline remain unchanged; current canonical files are never
+uploaded over the retained execution. Remote collection and cleanup serialize
+through Postgres across controller homes. A missing remote collector retains a
+pending save instead of treating an empty local cache as the agent's files.
 
 ## Cloud startup configuration
 

doc/plans/2026-10-31-zero-downtime-web-deploys.md

@@ -1,7 +1,30 @@
 # Zero-downtime web deployments
 
-Date: 2026-10-31. Status: runtime and Cloud overlap policy merged; qualification
-and existing-volume enrollment remain in progress. No hosted acceptance claim.
+Date: 2026-10-31. Status (2026-11-01 13:04 UTC): existing-volume staging enrollment
+is complete and its read/save/file-integrity checks pass. Active-agent recovery
+and the ten-deploy staging campaign remain unqualified. No production canary yet.
+
+Latest critical path: Core #18543 includes the runtime admission correction and
+owned immutable-cache retirement. Review exposed a resolved host-source fallback;
+its fix at `11d6eeefbb` passes all 122 workspace-preparation tests. The live test at
+`a18f6b85a3` gracefully stopped A and deleted its entire home in 50.4 seconds, then
+failed because B treated A's saved agent-directory cache path as execution
+identity. The remote execution did not meet the 420-second reattachment deadline.
+That failure remains failed evidence; all three owned Daytona sandboxes were
+subsequently confirmed absent and the temporary credential revoked. Recovery now
+needs a local projection rebuilt from the exact remote lease with its original
+save baseline preserved. Focused different-home/same-path/foreign-lease cases pass;
+full affected regressions and another live attempt are next.
+
+Cloud #1102 merged as `461c4aea28` and is serving on the staging controller.
+The original enrollment attempt completed using its original qualified image,
+a retained detached volume, and the migrated decision key. Its normal app boot
+became healthy; the post-enrollment probe passed 20 reads and three saves with no
+HTTP/WebSocket failures and verified all 34 files. The original browser tab
+reconnected and saved a new task title; lazy Agents navigation also succeeded.
+This is migrated-data/app continuity evidence, not a qualifying overlap or active
+agent result. Private receipts remain under the task's staging enrollment and
+hosted probe directories; no credential or live target identity belongs here.
 
 ## Outcome and endpoint
 

server/src/__tests__/agent-directory-working-copies.test.ts

@@ -8,13 +8,14 @@ import fs from "node:fs/promises";
 import { execFile as execFileCallback } from "node:child_process";
 import { promisify } from "node:util";
 import * as executionTargetTools from "@paperclipai/adapter-utils/execution-target";
+import * as checkpoints from "../services/agent-file-checkpoints.js";
 import * as ssh from "@paperclipai/adapter-utils/ssh";
 const execFile = promisify(execFileCallback);
 import os from "node:os";
 import path from "node:path";
 import { createHash, randomUUID } from "node:crypto";
 import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
-import { eq } from "drizzle-orm";
+import { eq, sql } from "drizzle-orm";
 import { agentFileStore, fileHash, inspectAgentFile, snapshotAgentFiles, MAX_AGENT_FILE_BYTES, MAX_AGENT_DIRECTORY_BYTES, MAX_AGENT_DIRECTORY_ENTRIES } from "../services/agent-file-store.js";
 import { agents, companies, authUsers, companyMemberships, principalPermissionGrants, heartbeatRuns, environmentLeases, environments, agentInstructionWorkingCopies, agentInstructionRevisions, agentInstructionHeads, createDb, fileStorageDb } from "@paperclipai/db";
 import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
@@ -902,6 +903,79 @@ describe("persistent agent directories", () => {
     expect((await fs.stat(active.localRoot)).isDirectory()).toBe(true);
   });
 
+  it.each(["new-home", "same-path-empty", "foreign-lease", "other-controller", "missing-collector"] as const)("recovers remote agent files after controller cache loss: %s", async scenario => {
+    const copy = await run({ warm: true });
+    const environmentId = randomUUID(), leaseId = randomUUID(), remoteCwd = "/fixture/retained-task";
+    const executionRoot = path.posix.join(remoteCwd, ".paperclip-runtime", "agent-files", agentId, copy.runId);
+    await db.insert(environments).values({ id: environmentId, name: environmentId, driver: "sandbox" });
+    await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId, heartbeatRunId: copy.runId,
+      provider: "daytona", providerLeaseId: "original-sandbox", status: "active", metadata: { remoteCwd } });
+    await db.update(agentInstructionWorkingCopies).set({ location: `remote:${environmentId}`, executionRoot,
+      receipt: { ...copy.receipt, cleanup: { leaseId, remoteCwd } } }).where(eq(agentInstructionWorkingCopies.runId, copy.runId));
+    const saved = (await copies.get(companyId, copy.runId))!;
+    const replacementHome = path.join(home, `replacement-${randomUUID()}`);
+    const remoteFiles = await fs.mkdtemp(path.join(home, "remote-checkpoint-"));
+    await fs.mkdir(path.join(remoteFiles, "files", "policy"), { recursive: true });
+    await fs.writeFile(path.join(remoteFiles, "files", entryFile), "Remote instructions changed during the turn");
+    await fs.writeFile(path.join(remoteFiles, "files", "unsaved.txt"), "Only the running sandbox has this");
+    await fs.rm(path.dirname(copy.localRoot), { recursive: true, force: true });
+    const capture = vi.spyOn(checkpoints, "captureAgentFileCheckpoint").mockResolvedValue({ directory: remoteFiles,
+      manifest: { version: 1, entries: [] }, stats: { hashedBytes: 0, copiedBytes: 0, copiedFiles: 0, scannedEntries: 0 }, cleanup: async () => {} });
+    const transport = vi.spyOn(executionTargetTools, "prepareAdapterExecutionTargetRuntime").mockImplementation(async options => ({
+      target: options.target!, workspaceRemoteDir: options.workspaceRemoteDir!, runtimeRootDir: options.runtimeRootDir!,
+      assetDirs: {}, additionalSourceDirs: {}, additionalSourceFailures: [], workspaceSyncSnapshot: null, restoreWorkspace: async () => {},
+    }));
+    const ephemeralBefore = process.env.PAPERCLIP_EPHEMERAL_SERVER;
+    process.env.PAPERCLIP_EPHEMERAL_SERVER = "true";
+    if (scenario === "new-home") process.env.PAPERCLIP_HOME = replacementHome;
+    const remote = { kind: "remote" as const, transport: "sandbox" as const, environmentId,
+      leaseId: scenario === "foreign-lease" ? randomUUID() : leaseId, remoteCwd, runner: { execute: vi.fn() } };
+    try {
+      if (scenario === "missing-collector") {
+        const retained = await copies.collectStopped({ companyId, runId: copy.runId });
+        expect(retained).toMatchObject({ state: "pending_collection", errorCode: "INSTRUCTION_REMOTE_COLLECTOR_UNAVAILABLE", baseHash: saved.baseHash });
+        expect(retained?.receipt?.baseline).toEqual(saved.receipt?.baseline);
+        expect(retained?.nextAttemptAt).toBeInstanceOf(Date);
+        expect(capture).not.toHaveBeenCalled(); expect(transport).not.toHaveBeenCalled();
+        return;
+      }
+      if (scenario === "other-controller") {
+        await db.transaction(async tx => {
+          await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`agent-directory:${companyId}:${agentId}`}, 0))`);
+          await expect(copies.prepare({ ...target(), runId: copy.runId, cwd: remoteCwd, target: remote, warm: true }))
+            .rejects.toThrow("Agent directory operation is already active");
+          expect(capture).not.toHaveBeenCalled(); expect(transport).not.toHaveBeenCalled();
+        });
+        return;
+      }
+      const preparing = copies.prepare({ ...target(), runId: copy.runId, cwd: remoteCwd, target: remote, warm: true });
+      if (scenario === "foreign-lease") {
+        await expect(preparing).rejects.toThrow("Agent directory recovery lease changed");
+        expect(capture).not.toHaveBeenCalled(); expect(transport).not.toHaveBeenCalled();
+        return;
+      }
+      const recovered = (await preparing)!;
+      expect(recovered.executionRoot).toBe(executionRoot);
+      expect(recovered.baseHash).toBe(saved.baseHash);
+      expect(recovered.receipt).toEqual(saved.receipt);
+      if (scenario === "new-home") {
+        expect(recovered.localRoot.startsWith(replacementHome)).toBe(true);
+        await expect(fs.stat(copy.localRoot)).rejects.toMatchObject({ code: "ENOENT" });
+      } else expect(recovered.localRoot).toBe(copy.localRoot);
+      expect(await fs.readFile(path.join(recovered.localRoot, "unsaved.txt"), "utf8")).toBe("Only the running sandbox has this");
+      expect(await fs.readFile(path.join(recovered.localRoot, entryFile), "utf8")).toBe("Remote instructions changed during the turn");
+      expect(capture).toHaveBeenCalledWith(expect.objectContaining({ executionRoot, previous: { version: 1, entries: [] }, target: remote }));
+      expect(transport).toHaveBeenCalledWith(expect.objectContaining({ workspaceLocalDir: recovered.localRoot, workspaceInboundMode: "adopt_remote" }));
+      await copies.prepare({ ...target(), runId: copy.runId, cwd: remoteCwd, target: remote, warm: true });
+      expect(capture).toHaveBeenCalledTimes(1);
+    } finally {
+      process.env.PAPERCLIP_HOME = home;
+      if (ephemeralBefore === undefined) delete process.env.PAPERCLIP_EPHEMERAL_SERVER;
+      else process.env.PAPERCLIP_EPHEMERAL_SERVER = ephemeralBefore;
+      capture.mockRestore(); transport.mockRestore();
+    }
+  });
+
   it("retains remote cleanup across restart and failed retries until the original lease is cleaned", async () => {
     const copy = await run();
     const environmentId = randomUUID(), leaseId = randomUUID(), remoteCwd = "/fixture/task";

server/src/services/agent-directory-working-copies.ts

@@ -3,7 +3,7 @@ import fs from "node:fs/promises";
 import path from "node:path";
 import { isDeepStrictEqual } from "node:util";
 import { and, eq, inArray, isNull, sql } from "drizzle-orm";
-import { agents, environmentLeases, environments, agentInstructionWorkingCopies as copies, type Db } from "@paperclipai/db";
+import { agents, environmentLeases, environments, agentInstructionWorkingCopies as copies, withDedicatedDbConnection, type Db } from "@paperclipai/db";
 import { syncDirectoryToSsh, restoreWorkspaceFromSshExecution } from "@paperclipai/adapter-utils/ssh";
 import { prepareAdapterExecutionTargetRuntime, runAdapterExecutionTargetShellCommand, type AdapterExecutionTarget, type PreparedAdapterExecutionTargetRuntime } from "@paperclipai/adapter-utils/execution-target";
 import { withDirectoryMergeLock, directorySnapshotSha256, parseDirectorySnapshot, serializeDirectorySnapshot, type DirectoryMergeLockOperation } from "@paperclipai/adapter-utils/workspace-restore-merge";
@@ -18,6 +18,7 @@ import type { Environment, EnvironmentLease } from "@paperclipai/shared";
 import { agentDirectoryBaselineDigest, agentDirectoryProbeProgram, observeLocalAgentDirectory } from "./agent-directory-probe.js";
 import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js";
 import { cachedAgentFileManifest, captureAgentFileCheckpoint, checkpointBaseline, checkpointSnapshot, type AgentFileManifest } from "./agent-file-checkpoints.js";
+import { withWorkspaceLockKeepalive } from "./native-runtime/native-workspace-lock-keepalive.js";
 import { logger } from "../middleware/logger.js";
 
 type Copy = typeof copies.$inferSelect;
@@ -54,7 +55,16 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
     });
   }
   const ownerFile = (row: Copy) => path.join(path.dirname(row.localRoot), "owner.json");
+  const disposableRemoteCopy = (row: Copy) => process.env.PAPERCLIP_EPHEMERAL_SERVER === "true" && row.location.startsWith("remote:");
+  function projectionRoot(row: Copy): string {
+    const origin = String(row.receipt?.materializationRunId ?? row.receipt?.directoryRunId ?? row.runId);
+    if (!/^[a-zA-Z0-9_-]{1,128}$/.test(origin)) throw conflict("Invalid agent directory owner");
+    return path.join(path.dirname(resolveManagedInstructionsRoot({ id: row.agentId, companyId: row.companyId, name: "", adapterConfig: {} })), "file-sync", "runs", origin, "live");
+  }
   async function owns(row: Copy) {
+    // Remote ownership is durable. A marker on a discarded controller cannot
+    // establish ownership of the retained sandbox on its replacement.
+    if (disposableRemoteCopy(row)) return row.state !== "superseded" && !row.receipt?.retainedByRunId;
     if (row.receipt?.warm !== true) return true;
     return await fs.readFile(ownerFile(row), "utf8").then(s => JSON.parse(s).runId === row.runId).catch(error => error.code === "ENOENT" && row.processStoppedAt !== null);
   }
@@ -159,7 +169,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
       ? path.posix.join(input.target.remoteCwd, ".paperclip-runtime", "agent-files", input.agentId, origin)
       : localRoot;
     const location = input.target?.kind === "remote" ? `remote:${input.target.environmentId ?? ""}` : "local";
-    if (row && (row.localRoot !== localRoot || row.executionRoot !== executionRoot || row.agentId !== input.agentId || row.location !== location)) throw conflict("Agent directory belongs to a different execution environment");
+    if (row && ((!disposableRemoteCopy(row) && row.localRoot !== localRoot) || row.executionRoot !== executionRoot || row.agentId !== input.agentId || row.location !== location)) throw conflict("Agent directory belongs to a different execution environment");
     if (row && !serialHeld) {
       // Restart preparation can replace a completed copy under the same run ID.
       // Join cleanup's lock before resetting its receipt or creating new bytes.
@@ -168,7 +178,10 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
       return serial(row, () => prepareCopy(input, true), "agent_directory_prepare");
     }
     if (row && !completed.has(row.state) && row.state !== "preparing") {
-      if (input.target?.kind === "remote" && !transports.has(key(row))) transports.set(key(row), await transport(row, input.target, true));
+      if (input.target?.kind === "remote") {
+        if (disposableRemoteCopy(row)) await restoreProjection(row, input.target);
+        if (!transports.has(key(row))) transports.set(key(row), await transport(row, input.target, true));
+      }
       return row;
     }
     // Register ownership before creating any bytes, so a crash during the copy
@@ -241,6 +254,32 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
     }
   }
 
+  async function restoreProjection(row: Copy, target: AdapterExecutionTarget) {
+    const exists = await fs.lstat(row.localRoot).then(stat => {
+      if (!stat.isDirectory() || stat.isSymbolicLink()) throw conflict("Agent directory projection is not a directory");
+      return true;
+    }, error => {
+      if (error.code === "ENOENT") return false;
+      throw error;
+    });
+    if (exists) return;
+    const lease = await ownedRemoteLease(row);
+    const cleanup = row.receipt?.cleanup as { leaseId?: string; remoteCwd?: string } | undefined;
+    if (target.kind !== "remote" || !lease || target.leaseId !== lease.id || lease.status !== "active"
+      || lease.releasedAt || hasRemoteTerminationReceipt(lease) || cleanup?.remoteCwd !== target.remoteCwd
+      || row.location !== `remote:${target.environmentId}`) throw conflict("Agent directory recovery lease changed");
+    // Fetch a full validated projection. The original save baseline remains in
+    // Postgres; neither current canonical files nor an empty cache are uploaded.
+    const captured = await captureAgentFileCheckpoint({ runId: row.runId, localRoot: row.localRoot,
+      executionRoot: row.executionRoot, previous: { version: 1, entries: [] }, target });
+    try {
+      await fs.cp(path.join(captured.directory, "files"), row.localRoot, { recursive: true });
+      if (row.receipt?.warm === true) await fs.writeFile(ownerFile(row), JSON.stringify({ runId: row.runId }));
+    } catch (error) {
+      await fs.rm(row.localRoot, { recursive: true, force: true });
+      throw error;
+    } finally { await captured.cleanup(); }
+  }
   async function retrieve(row: Copy, target?: AdapterExecutionTarget | null) {
     if (row.location !== "local") {
       let runtime = transports.get(key(row));
@@ -356,6 +395,14 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
   }
   async function checkpoint(row: Copy, target?: AdapterExecutionTarget | null, stopped = false): Promise<Copy> {
     if (!await owns(row) || row.state === "superseded") return row;
+    target ??= transports.get(key(row))?.target;
+    if (row.location !== "local" && target?.kind !== "remote") {
+      // A missing controller cache is not the remote directory. Retain the
+      // pending save until its exact remote collector or destruction proof exists.
+      return patch(row, { ...(stopped ? { state: "pending_collection", processStoppedAt: row.processStoppedAt ?? new Date() } : {}),
+        errorCode: "INSTRUCTION_REMOTE_COLLECTOR_UNAVAILABLE", errorMessage: "The remote agent directory is retained; its collector is unavailable. No save or discard is claimed.",
+        nextAttemptAt: stopped ? new Date(Date.now() + 30_000) : null });
+    }
     let failure: unknown;
     for (let attempt = 0; attempt < 3; attempt++) {
       let captured: Awaited<ReturnType<typeof captureAgentFileCheckpoint>> | undefined;
@@ -575,13 +622,29 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
     return result.exitCode === 0 && !result.timedOut;
   }
   async function serial<T>(row: Copy, fn: (current: Copy) => Promise<T>, operation: DirectoryMergeLockOperation): Promise<T> {
-    // Duplicate stop callbacks and restart recovery must not race while moving
-    // the stopped working copy. This lock is outside the writable tree.
-    return withDirectoryMergeLock(path.resolve(row.localRoot, "../../.."), async () => {
-      const current = await get(row.companyId, row.runId);
-      if (!current) throw notFound("Agent directory copy not found");
-      return fn(current);
-    }, process.env, operation);
+    const run = async () => {
+      const lockRoot = path.resolve(disposableRemoteCopy(row) ? projectionRoot(row) : row.localRoot, "../../..");
+      if (disposableRemoteCopy(row)) await fs.mkdir(lockRoot, { recursive: true, mode: 0o700 });
+      return withDirectoryMergeLock(lockRoot, async () => {
+        let current = await get(row.companyId, row.runId);
+        if (!current) throw notFound("Agent directory copy not found");
+        if (disposableRemoteCopy(current) && current.localRoot !== projectionRoot(current)) {
+          current = await patch(current, { localRoot: projectionRoot(current) });
+          if (current.localRoot !== projectionRoot(current)) throw conflict("Agent directory projection changed");
+          transports.delete(key(current));
+        }
+        return fn(current);
+      }, process.env, operation);
+    };
+    if (!disposableRemoteCopy(row)) return run();
+    // Different controllers have different cache roots. Serialize their active
+    // collection/cleanup operations in Postgres, without holding row locks over I/O.
+    return withDedicatedDbConnection(db, dedicated => dedicated.transaction(async tx => {
+      const [lock] = await tx.execute(sql`select pg_try_advisory_xact_lock(hashtextextended(${`agent-directory:${row.companyId}:${row.agentId}`}, 0)) as acquired,
+        (select setting::bigint from pg_settings where name = 'idle_in_transaction_session_timeout') as idle_timeout_ms`);
+      if (lock?.acquired !== true) throw conflict("Agent directory operation is already active");
+      return withWorkspaceLockKeepalive(tx as unknown as Db, Number(lock.idle_timeout_ms), run);
+    }));
   }
   return { prepare: (input: Parameters<typeof prepareCopy>[0]) => prepareCopy(input), hasChanges, adopt, canReuse, recoverUnavailable,
     recoverStoppedRemote: (row: Copy) => serial(row, current => current.receipt?.retainedByRunId ? Promise.resolve(current) : recoverStoppedRemote(current), "agent_directory_collect"),

server/src/services/agent-instruction-working-copies.ts

@@ -121,7 +121,7 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen
       if (signalsCleanup) await notifyDirectoryCleanup(tx, { companyId: row.companyId, runId: row.runId });
       if (signalsCollection) await notifyInstructionCollection(tx, { companyId: row.companyId, runId: row.runId });
       const [updated] = await tx.update(copies).set({ ...values, updatedAt: new Date() }).where(and(
-        scope(row.companyId, row.runId), eq(copies.state, row.state), eq(copies.attempts, row.attempts), eq(copies.baseHash, row.baseHash),
+        scope(row.companyId, row.runId), eq(copies.state, row.state), eq(copies.attempts, row.attempts), eq(copies.baseHash, row.baseHash), eq(copies.localRoot, row.localRoot),
         sql`${copies.receipt}->>'retainedByRunId' IS NOT DISTINCT FROM ${typeof row.receipt?.retainedByRunId === "string" ? row.receipt.retainedByRunId : null}`,
         requireDue ? or(isNull(copies.nextAttemptAt), lte(copies.nextAttemptAt, new Date())) : undefined, condition,
       )).returning();