fix(runner): fence warm directory successor ownership and recovery

Retain stable projectless scope, transfer collection to the exact successor lease, and preserve stopped remote files without restarting their sandbox. Validate complete remote directories and cover the composed DB/session lifecycle, retirement, and fresh preparation.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/agent-directory-probe.test.ts

@@ -3,8 +3,9 @@ import fs from "node:fs";
 import { join } from "node:path";
 import { tmpdir } from "node:os";
 import childProcess, { execFileSync } from "node:child_process";
+import { prepareAdapterExecutionTargetRuntime } from "@paperclipai/adapter-utils/execution-target";
 import { captureDirectorySnapshot } from "@paperclipai/adapter-utils/workspace-restore-merge";
-import { agentDirectoryBaselineDigest, agentDirectoryProbeProgram, observeLocalAgentDirectory, probeAgentDirectory } from "../services/agent-directory-probe.js";
+import { agentDirectoryBaselineDigest, agentDirectoryProbeProgram, observeLocalAgentDirectory, probeAgentDirectory, retireAgentDirectoryTransferScratch, agentDirectoryTransferCleanupProgram } from "../services/agent-directory-probe.js";
 
 describe("stable live agent directory observation", () => {
   let root: string;
@@ -22,6 +23,61 @@ describe("stable live agent directory observation", () => {
     const remote = JSON.parse(execFileSync(process.execPath, ["-e", agentDirectoryProbeProgram, root], { encoding: "utf8", env: {}, timeout: 10_000 }));
     expect(remote).toEqual(observed);
   });
+  it.runIf(process.platform === "darwin")("accepts only the observing Darwin host's fixed system aliases", () => {
+    const alias = root.replace(/^\/private(?=\/(tmp|var|etc)\/)/, "");
+    expect(alias).not.toBe(root);
+    const observed = probeAgentDirectory(root);
+    expect(probeAgentDirectory(alias)).toEqual(observed);
+    const remote = JSON.parse(execFileSync(process.execPath, ["-e", agentDirectoryProbeProgram, alias], { encoding: "utf8", env: {}, timeout: 10_000 }));
+    expect(remote).toEqual(observed);
+    // A Linux remote cannot inherit the controller's Darwin exception.
+    expect(() => execFileSync(process.execPath, ["-e", agentDirectoryProbeProgram.replace("platform:process.platform", "platform:'linux'"), alias],
+      { encoding: "utf8", env: {}, timeout: 10_000, stdio: "pipe" })).toThrow();
+    fs.symlinkSync(root, join(root, "redirect"));
+    expect(() => probeAgentDirectory(`${alias}/redirect`)).toThrow();
+  });
+  it("observes the complete agent directory after the actual remote transfer", async () => {
+    const remote = `${root}-remote`;
+    fs.mkdirSync(remote);
+    const target = { kind: "remote" as const, transport: "sandbox" as const, remoteCwd: remote,
+      runner: { execute: async (input: Parameters<import("@paperclipai/adapter-utils/command-managed-runtime").CommandManagedRuntimeRunner["execute"]>[0]) => ({
+        stdout: execFileSync(input.command, input.args ?? [], { cwd: input.cwd, input: input.stdin, encoding: "utf8",
+          env: { PATH: process.env.PATH, ...input.env }, timeout: input.timeoutMs, maxBuffer: 32 * 1024 * 1024 }),
+        stderr: "", exitCode: 0, signal: null, timedOut: false, pid: null, startedAt: new Date().toISOString(),
+      }) } };
+    let runtime: Awaited<ReturnType<typeof prepareAdapterExecutionTargetRuntime>> | undefined;
+    try {
+      const baseline = await captureDirectorySnapshot(root);
+      runtime = await prepareAdapterExecutionTargetRuntime({ target, runId: "probe-transfer", adapterKey: "agent-files",
+        workspaceLocalDir: root, workspaceRemoteDir: remote, syncWorkspace: true, workspaceBaseline: baseline,
+        workspaceGitSnapshot: null, workspaceFileMode: "all", workspaceExclude: [".paperclip-runtime", ".paperclip-runtime/**"] });
+      expect(fs.readdirSync(`${remote}/.paperclip-runtime`)).toEqual(["agent-files"]);
+      expect(fs.readdirSync(`${remote}/.paperclip-runtime/agent-files`)).toEqual([]);
+      execFileSync(process.execPath, ["-e", agentDirectoryTransferCleanupProgram, remote], { env: {}, timeout: 10_000 });
+      expect(() => probeAgentDirectory(remote)).not.toThrow();
+      expect(probeAgentDirectory(remote).digest).toBe(agentDirectoryBaselineDigest(baseline));
+    } finally { await runtime?.cleanupWorkspaceSnapshot?.(); fs.rmSync(remote, { recursive: true, force: true }); }
+  });
+  it.each(["file", "extra-directory", "symlink", "race"])("fails closed on %s in transfer-owned scratch without deleting contents", kind => {
+    const parent = join(root, ".paperclip-runtime"), scratch = join(parent, "agent-files");
+    fs.mkdirSync(scratch, { recursive: true });
+    const retained = join(parent, "retained.txt");
+    if (kind === "file") fs.writeFileSync(join(scratch, "retained.txt"), "keep");
+    if (kind === "extra-directory") fs.mkdirSync(join(parent, "unknown"));
+    if (kind === "symlink") { fs.rmdirSync(scratch); fs.symlinkSync(join(root, "notes"), scratch); }
+    if (kind === "race") {
+      const rmdir = fs.rmdirSync;
+      vi.spyOn(fs, "rmdirSync").mockImplementation(((directory: fs.PathLike) => {
+        if (directory === scratch) fs.writeFileSync(retained, "concurrent bytes");
+        return rmdir(directory);
+      }) as typeof rmdir);
+    }
+    expect(() => retireAgentDirectoryTransferScratch(root)).toThrow();
+    expect(fs.existsSync(parent)).toBe(true);
+    if (kind === "file") expect(fs.readFileSync(join(scratch, "retained.txt"), "utf8")).toBe("keep");
+    if (kind === "race") expect(fs.readFileSync(retained, "utf8")).toBe("concurrent bytes");
+    expect(fs.existsSync(join(root, "notes", "memory.bin"))).toBe(true);
+  });
   it("runs the local observation in its bounded credential-free child", async () => {
     expect(await observeLocalAgentDirectory(root)).toEqual(probeAgentDirectory(root));
   });

server/src/__tests__/agent-directory-working-copies.test.ts

@@ -540,13 +540,13 @@ describe("persistent agent directories", () => {
     expect(execute).toHaveBeenCalledTimes(2);
   });
 
-  it.each([false, true])("recovers a retained remote unchanged turn only with exact stop proof: %s", async stopped => {
+  it.each(["missing", "stopped", "destroyed"])("recovers a retained remote unchanged turn only with exact stop proof: %s", async stopped => {
     const copy = await run();
     const environmentId = randomUUID(), leaseId = randomUUID(), remoteCwd = "/fixture/task";
     const lease = { id: leaseId, companyId, environmentId, heartbeatRunId: copy.runId, provider: "daytona", providerLeaseId: "retained-sandbox" };
     await db.insert(environments).values({ id: environmentId, name: environmentId, driver: "sandbox" });
     await db.insert(environmentLeases).values({ ...lease, status: "released", releasedAt: new Date(), cleanupStatus: "success",
-      metadata: stopped ? { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "destroyed" }) } : {} });
+      metadata: stopped !== "missing" ? { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: stopped }) } : {} });
     await db.update(heartbeatRuns).set({ status: "succeeded", runtimeMode: "native" }).where(eq(heartbeatRuns.id, copy.runId));
     await db.update(agentInstructionWorkingCopies).set({ state: "unchanged_turn", location: `remote:${environmentId}`,
       executionRoot: path.posix.join(remoteCwd, ".paperclip-runtime", "agent-files", agentId, copy.runId),
@@ -556,19 +556,44 @@ describe("persistent agent directories", () => {
     expect((await copies.reportUnavailable(companyId, copy.runId))?.state).toBe("unchanged_turn");
     await copies.recoverStopped();
     const recovered = (await copies.get(companyId, copy.runId))!;
-    if (stopped) {
+    if (stopped === "destroyed") {
       expect(recovered).toMatchObject({ state: "unavailable", errorCode: "INSTRUCTION_COLLECTION_UNAVAILABLE", receipt: { cleanupPending: false } });
       expect(recovered.processStoppedAt).toBeInstanceOf(Date);
       await expect(fs.stat(copy.localRoot)).rejects.toMatchObject({ code: "ENOENT" });
       await copies.recoverStopped();
       expect((await copies.get(companyId, copy.runId))?.updatedAt).toEqual(recovered.updatedAt);
     } else {
-      expect(recovered).toMatchObject({ state: "unchanged_turn", processStoppedAt: null });
+      expect(recovered).toMatchObject({ state: stopped === "stopped" ? "pending_collection" : "unchanged_turn", processStoppedAt: null });
+      if (stopped === "stopped") {
+        expect(recovered.errorCode).toBe("INSTRUCTION_STOPPED_REMOTE_COLLECTION_PENDING");
+        expect(recovered.receipt?.baseline).toBeDefined();
+        await copies.release(companyId, copy.runId);
+        await copies.recoverCaptured();
+      }
       expect(await fs.readFile(path.join(copy.localRoot, entryFile), "utf8")).toBe(initial);
     }
     expect(execute).not.toHaveBeenCalled(); // Never restart a remote provider to recover bytes.
   });
 
+  it.each([0, 2])("preserves a legacy no-ID remote copy with %s matching leases", async count => {
+    const copy = await run();
+    const environmentId = randomUUID(), remoteCwd = "/fixture/task";
+    await db.insert(environments).values({ id: environmentId, name: `Legacy remote ${environmentId}`, driver: "sandbox" });
+    for (let index = 0; index < count; index++) await db.insert(environmentLeases).values({
+      companyId, environmentId, heartbeatRunId: copy.runId, provider: "daytona", providerLeaseId: `allocation-${index}` });
+    await db.update(agentInstructionWorkingCopies).set({ location: `remote:${environmentId}`,
+      executionRoot: path.posix.join(remoteCwd, ".paperclip-runtime", "agent-files", agentId, copy.runId),
+      receipt: { ...copy.receipt, cleanup: { remoteCwd } } }).where(eq(agentInstructionWorkingCopies.runId, copy.runId));
+    const execute = vi.fn();
+    copies = agentInstructionWorkingCopyService(db, { environmentRuntime: { execute } as unknown as EnvironmentRuntimeService });
+    const pending = await copies.collectStopped({ companyId, runId: copy.runId });
+    expect(pending).toMatchObject({ state: "pending_collection", attempts: 0, processStoppedAt: null, errorCode: "INSTRUCTION_REMOTE_LEASE_UNVERIFIED" });
+    expect(pending!.receipt?.baseline).toBeDefined();
+    await copies.release(companyId, copy.runId);
+    expect(await fs.readFile(path.join(copy.localRoot, entryFile), "utf8")).toBe(initial);
+    expect(execute).not.toHaveBeenCalled();
+  });
+
   it("finishes remote cleanup from a destruction receipt after the environment is deleted", async () => {
     const copy = await run();
     const environmentId = randomUUID(), leaseId = randomUUID(), remoteCwd = "/fixture/task";
@@ -650,10 +675,14 @@ describe("persistent agent directories", () => {
         }
       },
     };
-    const executionTarget = { kind: "remote" as const, transport: "sandbox" as const, environmentId: randomUUID(), remoteCwd, runner };
+    const executionTarget = { kind: "remote" as const, transport: "sandbox" as const, environmentId: randomUUID(), leaseId: "", remoteCwd, runner };
+    await db.insert(environments).values({ id: executionTarget.environmentId, name: "Fixture sandbox", driver: "sandbox" });
     const prepare = async () => {
       const runId = randomUUID();
       await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", responsibleUserId: userId });
+      executionTarget.leaseId = randomUUID();
+      await db.insert(environmentLeases).values({ id: executionTarget.leaseId, companyId, environmentId: executionTarget.environmentId,
+        heartbeatRunId: runId, provider: "daytona", providerLeaseId: "fixture-allocation" });
       return (await copies.prepare({ ...target(), runId, cwd: home, target: executionTarget }))!;
     };
     await fs.mkdir(path.join(root, "build"));
@@ -696,6 +725,9 @@ describe("persistent agent directories", () => {
       await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", responsibleUserId: userId });
       const target = { kind: "remote" as const, transport: "ssh" as const, environmentId: randomUUID(), remoteCwd,
         spec: { host: "unused.invalid", port: 22, username: "test", remoteCwd } };
+      await db.insert(environments).values({ id: target.environmentId, name: "Fixture SSH", driver: "ssh" });
+      // Legacy no-ID receipt: only the unique run/environment lease authorizes retrieval.
+      await db.insert(environmentLeases).values({ companyId, environmentId: target.environmentId, heartbeatRunId: runId, provider: "ssh", providerLeaseId: "fixture-host" });
       const copy = (await copies.prepare({ companyId, agentId, runId, cwd: home, target }))!;
       expect(stage).toHaveBeenCalledWith(expect.objectContaining({ remoteDir: copy.executionRoot }));
       expect(await fs.readFile(path.join(copy.executionRoot, entryFile), "utf8")).toBe(initial);