fix: preserve durable exe sessions across checkpoint and continuation

Keep VM attestation out of configuration freshness, defer recovery while the live heartbeat owns finalization, exclude generated OpenCode launch state from checkpoints, and safely restage read-only runtime bundles.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts

@@ -2476,12 +2476,16 @@ function sessionRoot(
   runtimeDirectory: string,
   normalizedSessionId: string,
 ): string {
+  return join(resolve(runtimeDirectory), opencodeRuntimeSessionDirectoryName(normalizedSessionId));
+}
+
+export function opencodeRuntimeSessionDirectoryName(normalizedSessionId: string): string {
   const safe = normalizedSessionId
     .replace(/[^a-zA-Z0-9._-]/g, "_")
     .slice(0, 120);
   if (!safe || safe === "." || safe === "..")
     throw new Error("Invalid normalized OpenCode session id");
-  return join(resolve(runtimeDirectory), safe);
+  return safe;
 }
 
 function validateWorkspace(value: string): string {

packages/paperclip-runner/src/index.ts

@@ -18,6 +18,7 @@ export {
 } from "./backends/native-backend-factory.js";
 export {
   OpenCodeServerDriver,
+  opencodeRuntimeSessionDirectoryName,
   type OpenCodeServerDriverOptions,
 } from "./drivers/opencode/opencode-server-driver.js";
 export {

packages/plugins/sandbox-providers/exe-dev/README.md

@@ -81,6 +81,9 @@ but a changed remote file takes precedence when the same file also changed on th
 host. Use isolated worktrees for concurrent editing; this is not a conflict merge.
 Native runs additionally use
 the existing verified harness/session checkpoint and restart-recovery contract.
+OpenCode checkpoints preserve its session database and history while excluding
+regenerated configuration (including launch credentials and npm executable
+aliases) and caches. Codex launch credentials and scratch aliases are also excluded.
 The existing exclusions still apply, including dependencies, generated/cache
 folders, ignored files, and runtime scratch.
 

server/src/__tests__/exe-environment-resource.test.ts

@@ -37,6 +37,19 @@ const support = await getEmbeddedPostgresTestSupport();
     await Promise.all(Array.from({ length: 8 }, () => bindEnvironmentResource(db, id, companyId, binding)));
     expect(await readEnvironmentResourceBinding(db, id, companyId)).toEqual(binding);
   });
+  it("keeps the configuration revision stable while attesting VM identity", async () => {
+    const { id, companyId, binding } = await fixture();
+    const revision = new Date("2020-01-01T00:00:00Z");
+    await db.update(environments).set({ updatedAt: revision }).where(eq(environments.id, id));
+    await bindEnvironmentResource(db, id, companyId, binding);
+    await bindEnvironmentResource(db, id, companyId, binding);
+    const [bound] = await db.select().from(environments).where(eq(environments.id, id));
+    expect(bound.updatedAt).toEqual(revision);
+    expect(await readEnvironmentResourceBinding(db, id, companyId)).toEqual(binding);
+    await environmentService(db).update(id, { config: { provider: "exe-dev", vmName: "operator-change" } });
+    const [edited] = await db.select().from(environments).where(eq(environments.id, id));
+    expect(edited.updatedAt.getTime()).toBeGreaterThan(revision.getTime());
+  });
   it("allows only one identity when competing first acquisitions disagree", async () => {
     const { id, companyId, binding } = await fixture();
     const results = await Promise.allSettled([

server/src/__tests__/native-finalization-recovery.test.ts

@@ -213,6 +213,17 @@ describe("P6-16/P6-25/P6-28 native finalization recovery", () => {
     await temporary.cleanup();
   });
 
+  it("leaves finalization to an active heartbeat instead of racing its copyback", async () => {
+    const [before] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId));
+    const isRunActive = vi.fn((candidate: string) => candidate === runId);
+    await expect(reconcileNativeFinalizations(db, [runId], { isRunActive })).resolves.toEqual([]);
+    expect(isRunActive).toHaveBeenCalledWith(runId);
+    const [after] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId));
+    expect(after).toEqual(before);
+    // The following recovery tests omit the live owner and must still settle
+    // these persisted runs, as a new controller does after restart.
+  });
+
   it("fails closed into bounded named recovery without consulting the live flag or falling back", async () => {
     await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([
       expect.objectContaining({ phase: "retryable_failure", failureCode: "native_finalization_invalid" }),

server/src/services/environment-resource-binding.ts

@@ -28,7 +28,9 @@ export async function bindEnvironmentResource(db: Db, environmentId: string, com
   const encoded = JSON.stringify(binding);
   const rows = await db.update(environments).set({
     metadata: sql`coalesce(${environments.metadata}, '{}'::jsonb) || jsonb_build_object('environmentResourceBinding', ${encoded}::jsonb)`,
-    updatedAt: new Date(),
+    // Binding attestation is runtime state. updatedAt is also the configuration
+    // revision used for session compatibility; changing it would rotate every
+    // agent session when this VM is first bound or another lease is acquired.
   }).where(and(eq(environments.id, environmentId), sql`(${environments.metadata}->'environmentResourceBinding' IS NULL OR ${environments.metadata}->'environmentResourceBinding' = ${encoded}::jsonb)`)).returning({ id: environments.id });
   if (rows.length !== 1) throw conflict("Durable VM identity changed; explicit recovery to a new environment is required");
 }

server/src/services/heartbeat.ts

@@ -14758,6 +14758,7 @@ export function heartbeatService(
     // whether any provider authority needs to be reopened.
     await reconcileNativeFinalizations(db, undefined, {
       environmentRuntime,
+      isRunActive: (runId) => activeRunExecutions.has(runId),
       onWorkspaceSettled: settleRecoveredNativeWorkspace,
     });
     scheduleRetainedNativeSessionCleanup();
@@ -18627,6 +18628,7 @@ export function heartbeatService(
     // mode, never the current feature flag.
     await reconcileNativeFinalizations(db, undefined, {
       environmentRuntime,
+      isRunActive: (runId) => activeRunExecutions.has(runId),
       onWorkspaceSettled: settleRecoveredNativeWorkspace,
     }).catch((error) => {
       logger.warn(

server/src/services/native-runtime/native-finalization-reconciler.ts

@@ -538,6 +538,8 @@ export async function reconcileNativeFinalizations(
   runIds?: string[],
   options: {
     environmentRuntime?: EnvironmentRuntimeService;
+    /** The live heartbeat owns its copyback/status suffix until its finally exits. */
+    isRunActive?: (runId: string) => boolean;
     onWorkspaceSettled?: (input: {
       runId: string;
       companyId: string;
@@ -594,6 +596,10 @@ export async function reconcileNativeFinalizations(
     ));
   const results = [];
   for (const row of rows) {
+    // A provider can persist its result before the heartbeat finishes exporting
+    // files. Concurrent recovery would reuse and remove that same export archive.
+    // Startup has no live owner, so crashed runs still follow durable recovery.
+    if (options.isRunActive?.(row.runId)) continue;
     const pendingEffects = row.decisionId
       ? await db.select({ id: statusDecisionEffects.id }).from(statusDecisionEffects).where(and(
           eq(statusDecisionEffects.companyId, row.companyId),

server/src/services/native-runtime/native-session-executor.test.ts

@@ -1,6 +1,7 @@
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import {
   access,
+  chmod,
   lstat,
   mkdir,
   mkdtemp,
@@ -1578,6 +1579,39 @@ describe("remote provider checkpoint snapshots", () => {
     expect(syncOut).toHaveBeenCalledOnce();
   });
 
+  it("persists OpenCode history while omitting generated config, credentials, cache, and executable aliases", async () => {
+    const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-checkpoint-"));
+    const sourcePath = join(root, "source");
+    const targetPath = join(root, "durable");
+    const sessionDirectory = "session";
+    try {
+      for (const directory of ["data", "config", "cache"]) {
+        await mkdir(join(sourcePath, sessionDirectory, directory), { recursive: true });
+      }
+      await writeFile(join(sourcePath, sessionDirectory, "data", "session.db"), "resume-history");
+      await writeFile(join(sourcePath, sessionDirectory, "config", "opencode.json"), "launch-secret");
+      await symlink("/outside/executable", join(sourcePath, sessionDirectory, "config", "npm-bin"));
+      await symlink("/outside/cache", join(sourcePath, sessionDirectory, "cache", "bin"));
+      const profile = resolveNativeHarnessPersistenceProfile({
+        provider: { kind: "opencode" },
+        session: { normalizedSessionId: "session", driverKind: "opencode_server" },
+      } as NativeExecutionInputV1);
+      const directory = profile.directories.find((entry) => entry.name === "opencode")!;
+      const execute = vi.fn(async ({ command, args }: { command: string; args: string[] }) => ({
+        exitCode: 0, timedOut: false, stderr: "",
+        stdout: execFileSync(command, args, { encoding: "utf8" }),
+      }));
+      await syncRemoteRunnerDirectoryOut({ runner: { execute } as never,
+        sourcePath, targetPath, mode: 0o700, excludeEntries: directory.excludeEntries });
+      expect(await readFile(join(targetPath, sessionDirectory, "data", "session.db"), "utf8")).toBe("resume-history");
+      await expect(access(join(targetPath, sessionDirectory, "config"))).rejects.toThrow();
+      await expect(access(join(targetPath, sessionDirectory, "cache"))).rejects.toThrow();
+      expect(await readFile(join(sourcePath, sessionDirectory, "config", "opencode.json"), "utf8")).toBe("launch-secret");
+    } finally {
+      await rm(root, { recursive: true, force: true });
+    }
+  });
+
   it("rejects unsafe relative checkpoint exclusions", async () => {
     const execute = vi.fn().mockResolvedValue({
       exitCode: 0,
@@ -1643,6 +1677,42 @@ describe("remote provider checkpoint snapshots", () => {
 });
 
 describe("remote provider checkpoint restores", () => {
+  it("replaces read-only runtime instructions on a reused VM without writing through file aliases", async () => {
+    const root = await mkdtemp(join(tmpdir(), "paperclip-readonly-restage-"));
+    const sourcePath = join(root, "source");
+    const targetPath = join(root, "remote");
+    const outside = join(root, "outside.txt");
+    try {
+      await mkdir(join(sourcePath, "nested"), { recursive: true });
+      await writeFile(join(sourcePath, "nested", "AGENTS.md"), "first");
+      await chmod(join(sourcePath, "nested", "AGENTS.md"), 0o444);
+      await chmod(join(sourcePath, "nested"), 0o555);
+      const execute = vi.fn(async ({ command, args, stdin }: { command: string; args: string[]; stdin?: string }) => ({
+        exitCode: 0, timedOut: false, stderr: "",
+        stdout: execFileSync(command, args, { input: stdin, encoding: "utf8" }),
+      }));
+      const input = { target: {} as never, runner: { execute } as never, sourcePath, targetPath, mode: 0o555 };
+      await stageRemoteRunnerDirectory(input);
+      await chmod(join(sourcePath, "nested", "AGENTS.md"), 0o644);
+      await writeFile(join(sourcePath, "nested", "AGENTS.md"), "revised");
+      await chmod(join(sourcePath, "nested", "AGENTS.md"), 0o444);
+      await stageRemoteRunnerDirectory(input);
+      expect(await readFile(join(targetPath, "nested", "AGENTS.md"), "utf8")).toBe("revised");
+      expect((await lstat(targetPath)).mode & 0o777).toBe(0o555);
+      await writeFile(outside, "untouched");
+      await chmod(join(targetPath, "nested"), 0o755);
+      await rm(join(targetPath, "nested", "AGENTS.md"));
+      await symlink(outside, join(targetPath, "nested", "AGENTS.md"));
+      await chmod(join(targetPath, "nested"), 0o555);
+      await stageRemoteRunnerDirectory(input);
+      expect(await readFile(outside, "utf8")).toBe("untouched");
+      expect((await lstat(join(targetPath, "nested", "AGENTS.md"))).isSymbolicLink()).toBe(false);
+    } finally {
+      execFileSync("chmod", ["-R", "u+w", root]);
+      await rm(root, { recursive: true, force: true });
+    }
+  });
+
   it("does not upload excluded Codex scratch trees or credentials", async () => {
     const sourcePath = await mkdtemp(
       join(tmpdir(), "paperclip-codex-restore-source-"),

server/src/services/native-runtime/native-session-executor.ts

@@ -59,6 +59,7 @@ import {
   completeRetainedNativeSessionCleanup,
   completeTerminatedLocalNativeSessionCleanup,
   acpxRuntimeSessionDirectoryName,
+  opencodeRuntimeSessionDirectoryName,
   createNativeSessionBackend,
   createRunnerdCodexTransport,
   defaultCapabilityRunnerdBinary,
@@ -5036,7 +5037,12 @@ export function resolveNativeHarnessPersistenceProfile(
         ? {
             name: "opencode",
             location: "filesystem",
-            excludeEntries: [],
+            // OpenCode regenerates config (including MCP credentials and npm
+            // executable symlinks) and cache on launch. Its resumable session
+            // database lives in data, which must remain in the checkpoint.
+            excludeEntries: ["config", "cache"].map(
+              (entry) => `${opencodeRuntimeSessionDirectoryName(nativeSessionKey(execution))}/${entry}`,
+            ),
           }
         : execution.provider.kind === "acpx"
           ? {
@@ -9114,7 +9120,7 @@ export async function stageRemoteRunnerDirectory(input: {
         const archive = execFileSync(
           "tar",
           [...excludeArgs, "-czf", "-", "-C", input.sourcePath, "."],
-          { encoding: "buffer", maxBuffer: 64 * 1024 * 1024 },
+          { encoding: "buffer", maxBuffer: 64 * 1024 * 1024, env: { ...process.env, COPYFILE_DISABLE: "1" } },
         );
         execFileSync("tar", ["-xzf", "-", "-C", stagingRoot], {
           input: archive,
@@ -9143,11 +9149,18 @@ export async function stageRemoteRunnerDirectory(input: {
   const archive = execFileSync(
     "tar",
     [...excludeArgs, "-czf", "-", "-C", input.sourcePath, "."],
-    { encoding: "buffer", maxBuffer: 64 * 1024 * 1024 },
+    { encoding: "buffer", maxBuffer: 64 * 1024 * 1024, env: { ...process.env, COPYFILE_DISABLE: "1" } },
   );
   const escapedTarget = input.targetPath.replaceAll("'", "'\\''");
+  // Runtime instruction/skill snapshots are read-only after staging. A durable
+  // VM reuses this path on the next turn: permit replacing its entries without
+  // following symlinks into another workspace, then restore archive modes.
+  const prepareReadonly = input.mode === 0o555
+    ? `test ! -L '${escapedTarget}' && find '${escapedTarget}' -type d -exec chmod u+w {} + && `
+    : "";
   const script =
     `umask 077; mkdir -p '${escapedTarget}' && ` +
+    prepareReadonly +
     `base64 -d | tar -xzf - -C '${escapedTarget}' && ` +
     `chmod ${input.mode.toString(8)} '${escapedTarget}'`;
   const result = await input.runner.execute({

server/src/vendor/paperclip-runner/index.ts

@@ -83,6 +83,7 @@ export const NATIVE_RUNTIME_ASSET_SCHEMA = runner.NATIVE_RUNTIME_ASSET_SCHEMA;
 export const PAPERCLIP_EXECUTION_PROMPT = runner.PAPERCLIP_EXECUTION_PROMPT;
 export const PAPERCLIP_EXECUTION_PROMPT_REVISION =
   runner.PAPERCLIP_EXECUTION_PROMPT_REVISION;
+export const opencodeRuntimeSessionDirectoryName = runner.opencodeRuntimeSessionDirectoryName;
 export const acpxRuntimeSessionDirectoryName =
   runner.acpxRuntimeSessionDirectoryName;
 export const canonicalNativeRuntimeContextDigest =