From 9247d6f6bbef2aa66c4c341444182dfdc8df051e Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 17 Sep 2026 10:54:37 -0500 Subject: [PATCH] fix: preserve durable exe sessions across checkpoint and continuation Keep VM attestation out of configuration freshness, defer recovery while the live heartbeat owns finalization, exclude generated OpenCode launch state from checkpoints, and safely restage read-only runtime bundles. Co-Authored-By: Paperclip --- .../opencode/opencode-server-driver.ts | 6 +- packages/paperclip-runner/src/index.ts | 1 + .../sandbox-providers/exe-dev/README.md | 3 + .../exe-environment-resource.test.ts | 13 ++++ .../native-finalization-recovery.test.ts | 11 +++ .../services/environment-resource-binding.ts | 4 +- server/src/services/heartbeat.ts | 2 + .../native-finalization-reconciler.ts | 6 ++ .../native-session-executor.test.ts | 70 +++++++++++++++++++ .../native-runtime/native-session-executor.ts | 19 ++++- server/src/vendor/paperclip-runner/index.ts | 1 + 11 files changed, 131 insertions(+), 5 deletions(-) diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts index 6e309cc09c7..4233e5ce594 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts @@ -2476,12 +2476,16 @@ function sessionRoot( runtimeDirectory: string, normalizedSessionId: string, ): string { + return join(resolve(runtimeDirectory), opencodeRuntimeSessionDirectoryName(normalizedSessionId)); +} + +export function opencodeRuntimeSessionDirectoryName(normalizedSessionId: string): string { const safe = normalizedSessionId .replace(/[^a-zA-Z0-9._-]/g, "_") .slice(0, 120); if (!safe || safe === "." || safe === "..") throw new Error("Invalid normalized OpenCode session id"); - return join(resolve(runtimeDirectory), safe); + return safe; } function validateWorkspace(value: string): string { diff --git a/packages/paperclip-runner/src/index.ts b/packages/paperclip-runner/src/index.ts index 7aeec692b85..ef9130167e1 100644 --- a/packages/paperclip-runner/src/index.ts +++ b/packages/paperclip-runner/src/index.ts @@ -18,6 +18,7 @@ export { } from "./backends/native-backend-factory.js"; export { OpenCodeServerDriver, + opencodeRuntimeSessionDirectoryName, type OpenCodeServerDriverOptions, } from "./drivers/opencode/opencode-server-driver.js"; export { diff --git a/packages/plugins/sandbox-providers/exe-dev/README.md b/packages/plugins/sandbox-providers/exe-dev/README.md index 651c0f57599..8622db19593 100644 --- a/packages/plugins/sandbox-providers/exe-dev/README.md +++ b/packages/plugins/sandbox-providers/exe-dev/README.md @@ -81,6 +81,9 @@ but a changed remote file takes precedence when the same file also changed on th host. Use isolated worktrees for concurrent editing; this is not a conflict merge. Native runs additionally use the existing verified harness/session checkpoint and restart-recovery contract. +OpenCode checkpoints preserve its session database and history while excluding +regenerated configuration (including launch credentials and npm executable +aliases) and caches. Codex launch credentials and scratch aliases are also excluded. The existing exclusions still apply, including dependencies, generated/cache folders, ignored files, and runtime scratch. diff --git a/server/src/__tests__/exe-environment-resource.test.ts b/server/src/__tests__/exe-environment-resource.test.ts index f448b67620a..ba8f618372b 100644 --- a/server/src/__tests__/exe-environment-resource.test.ts +++ b/server/src/__tests__/exe-environment-resource.test.ts @@ -37,6 +37,19 @@ const support = await getEmbeddedPostgresTestSupport(); await Promise.all(Array.from({ length: 8 }, () => bindEnvironmentResource(db, id, companyId, binding))); expect(await readEnvironmentResourceBinding(db, id, companyId)).toEqual(binding); }); + it("keeps the configuration revision stable while attesting VM identity", async () => { + const { id, companyId, binding } = await fixture(); + const revision = new Date("2020-01-01T00:00:00Z"); + await db.update(environments).set({ updatedAt: revision }).where(eq(environments.id, id)); + await bindEnvironmentResource(db, id, companyId, binding); + await bindEnvironmentResource(db, id, companyId, binding); + const [bound] = await db.select().from(environments).where(eq(environments.id, id)); + expect(bound.updatedAt).toEqual(revision); + expect(await readEnvironmentResourceBinding(db, id, companyId)).toEqual(binding); + await environmentService(db).update(id, { config: { provider: "exe-dev", vmName: "operator-change" } }); + const [edited] = await db.select().from(environments).where(eq(environments.id, id)); + expect(edited.updatedAt.getTime()).toBeGreaterThan(revision.getTime()); + }); it("allows only one identity when competing first acquisitions disagree", async () => { const { id, companyId, binding } = await fixture(); const results = await Promise.allSettled([ diff --git a/server/src/__tests__/native-finalization-recovery.test.ts b/server/src/__tests__/native-finalization-recovery.test.ts index 90d0be71741..bfd3f993c21 100644 --- a/server/src/__tests__/native-finalization-recovery.test.ts +++ b/server/src/__tests__/native-finalization-recovery.test.ts @@ -213,6 +213,17 @@ describe("P6-16/P6-25/P6-28 native finalization recovery", () => { await temporary.cleanup(); }); + it("leaves finalization to an active heartbeat instead of racing its copyback", async () => { + const [before] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + const isRunActive = vi.fn((candidate: string) => candidate === runId); + await expect(reconcileNativeFinalizations(db, [runId], { isRunActive })).resolves.toEqual([]); + expect(isRunActive).toHaveBeenCalledWith(runId); + const [after] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + expect(after).toEqual(before); + // The following recovery tests omit the live owner and must still settle + // these persisted runs, as a new controller does after restart. + }); + it("fails closed into bounded named recovery without consulting the live flag or falling back", async () => { await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([ expect.objectContaining({ phase: "retryable_failure", failureCode: "native_finalization_invalid" }), diff --git a/server/src/services/environment-resource-binding.ts b/server/src/services/environment-resource-binding.ts index 3767691bf1f..3ffc9ccb546 100644 --- a/server/src/services/environment-resource-binding.ts +++ b/server/src/services/environment-resource-binding.ts @@ -28,7 +28,9 @@ export async function bindEnvironmentResource(db: Db, environmentId: string, com const encoded = JSON.stringify(binding); const rows = await db.update(environments).set({ metadata: sql`coalesce(${environments.metadata}, '{}'::jsonb) || jsonb_build_object('environmentResourceBinding', ${encoded}::jsonb)`, - updatedAt: new Date(), + // Binding attestation is runtime state. updatedAt is also the configuration + // revision used for session compatibility; changing it would rotate every + // agent session when this VM is first bound or another lease is acquired. }).where(and(eq(environments.id, environmentId), sql`(${environments.metadata}->'environmentResourceBinding' IS NULL OR ${environments.metadata}->'environmentResourceBinding' = ${encoded}::jsonb)`)).returning({ id: environments.id }); if (rows.length !== 1) throw conflict("Durable VM identity changed; explicit recovery to a new environment is required"); } diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index abb5961d5f2..0b81db43b3a 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -14758,6 +14758,7 @@ export function heartbeatService( // whether any provider authority needs to be reopened. await reconcileNativeFinalizations(db, undefined, { environmentRuntime, + isRunActive: (runId) => activeRunExecutions.has(runId), onWorkspaceSettled: settleRecoveredNativeWorkspace, }); scheduleRetainedNativeSessionCleanup(); @@ -18627,6 +18628,7 @@ export function heartbeatService( // mode, never the current feature flag. await reconcileNativeFinalizations(db, undefined, { environmentRuntime, + isRunActive: (runId) => activeRunExecutions.has(runId), onWorkspaceSettled: settleRecoveredNativeWorkspace, }).catch((error) => { logger.warn( diff --git a/server/src/services/native-runtime/native-finalization-reconciler.ts b/server/src/services/native-runtime/native-finalization-reconciler.ts index c88479166bb..5c8344dfad7 100644 --- a/server/src/services/native-runtime/native-finalization-reconciler.ts +++ b/server/src/services/native-runtime/native-finalization-reconciler.ts @@ -538,6 +538,8 @@ export async function reconcileNativeFinalizations( runIds?: string[], options: { environmentRuntime?: EnvironmentRuntimeService; + /** The live heartbeat owns its copyback/status suffix until its finally exits. */ + isRunActive?: (runId: string) => boolean; onWorkspaceSettled?: (input: { runId: string; companyId: string; @@ -594,6 +596,10 @@ export async function reconcileNativeFinalizations( )); const results = []; for (const row of rows) { + // A provider can persist its result before the heartbeat finishes exporting + // files. Concurrent recovery would reuse and remove that same export archive. + // Startup has no live owner, so crashed runs still follow durable recovery. + if (options.isRunActive?.(row.runId)) continue; const pendingEffects = row.decisionId ? await db.select({ id: statusDecisionEffects.id }).from(statusDecisionEffects).where(and( eq(statusDecisionEffects.companyId, row.companyId), diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 4f40c3e0904..d97f7568488 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { access, + chmod, lstat, mkdir, mkdtemp, @@ -1578,6 +1579,39 @@ describe("remote provider checkpoint snapshots", () => { expect(syncOut).toHaveBeenCalledOnce(); }); + it("persists OpenCode history while omitting generated config, credentials, cache, and executable aliases", async () => { + const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-checkpoint-")); + const sourcePath = join(root, "source"); + const targetPath = join(root, "durable"); + const sessionDirectory = "session"; + try { + for (const directory of ["data", "config", "cache"]) { + await mkdir(join(sourcePath, sessionDirectory, directory), { recursive: true }); + } + await writeFile(join(sourcePath, sessionDirectory, "data", "session.db"), "resume-history"); + await writeFile(join(sourcePath, sessionDirectory, "config", "opencode.json"), "launch-secret"); + await symlink("/outside/executable", join(sourcePath, sessionDirectory, "config", "npm-bin")); + await symlink("/outside/cache", join(sourcePath, sessionDirectory, "cache", "bin")); + const profile = resolveNativeHarnessPersistenceProfile({ + provider: { kind: "opencode" }, + session: { normalizedSessionId: "session", driverKind: "opencode_server" }, + } as NativeExecutionInputV1); + const directory = profile.directories.find((entry) => entry.name === "opencode")!; + const execute = vi.fn(async ({ command, args }: { command: string; args: string[] }) => ({ + exitCode: 0, timedOut: false, stderr: "", + stdout: execFileSync(command, args, { encoding: "utf8" }), + })); + await syncRemoteRunnerDirectoryOut({ runner: { execute } as never, + sourcePath, targetPath, mode: 0o700, excludeEntries: directory.excludeEntries }); + expect(await readFile(join(targetPath, sessionDirectory, "data", "session.db"), "utf8")).toBe("resume-history"); + await expect(access(join(targetPath, sessionDirectory, "config"))).rejects.toThrow(); + await expect(access(join(targetPath, sessionDirectory, "cache"))).rejects.toThrow(); + expect(await readFile(join(sourcePath, sessionDirectory, "config", "opencode.json"), "utf8")).toBe("launch-secret"); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + it("rejects unsafe relative checkpoint exclusions", async () => { const execute = vi.fn().mockResolvedValue({ exitCode: 0, @@ -1643,6 +1677,42 @@ describe("remote provider checkpoint snapshots", () => { }); describe("remote provider checkpoint restores", () => { + it("replaces read-only runtime instructions on a reused VM without writing through file aliases", async () => { + const root = await mkdtemp(join(tmpdir(), "paperclip-readonly-restage-")); + const sourcePath = join(root, "source"); + const targetPath = join(root, "remote"); + const outside = join(root, "outside.txt"); + try { + await mkdir(join(sourcePath, "nested"), { recursive: true }); + await writeFile(join(sourcePath, "nested", "AGENTS.md"), "first"); + await chmod(join(sourcePath, "nested", "AGENTS.md"), 0o444); + await chmod(join(sourcePath, "nested"), 0o555); + const execute = vi.fn(async ({ command, args, stdin }: { command: string; args: string[]; stdin?: string }) => ({ + exitCode: 0, timedOut: false, stderr: "", + stdout: execFileSync(command, args, { input: stdin, encoding: "utf8" }), + })); + const input = { target: {} as never, runner: { execute } as never, sourcePath, targetPath, mode: 0o555 }; + await stageRemoteRunnerDirectory(input); + await chmod(join(sourcePath, "nested", "AGENTS.md"), 0o644); + await writeFile(join(sourcePath, "nested", "AGENTS.md"), "revised"); + await chmod(join(sourcePath, "nested", "AGENTS.md"), 0o444); + await stageRemoteRunnerDirectory(input); + expect(await readFile(join(targetPath, "nested", "AGENTS.md"), "utf8")).toBe("revised"); + expect((await lstat(targetPath)).mode & 0o777).toBe(0o555); + await writeFile(outside, "untouched"); + await chmod(join(targetPath, "nested"), 0o755); + await rm(join(targetPath, "nested", "AGENTS.md")); + await symlink(outside, join(targetPath, "nested", "AGENTS.md")); + await chmod(join(targetPath, "nested"), 0o555); + await stageRemoteRunnerDirectory(input); + expect(await readFile(outside, "utf8")).toBe("untouched"); + expect((await lstat(join(targetPath, "nested", "AGENTS.md"))).isSymbolicLink()).toBe(false); + } finally { + execFileSync("chmod", ["-R", "u+w", root]); + await rm(root, { recursive: true, force: true }); + } + }); + it("does not upload excluded Codex scratch trees or credentials", async () => { const sourcePath = await mkdtemp( join(tmpdir(), "paperclip-codex-restore-source-"), diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 2173f6b6f99..9ef7098e0bf 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -59,6 +59,7 @@ import { completeRetainedNativeSessionCleanup, completeTerminatedLocalNativeSessionCleanup, acpxRuntimeSessionDirectoryName, + opencodeRuntimeSessionDirectoryName, createNativeSessionBackend, createRunnerdCodexTransport, defaultCapabilityRunnerdBinary, @@ -5036,7 +5037,12 @@ export function resolveNativeHarnessPersistenceProfile( ? { name: "opencode", location: "filesystem", - excludeEntries: [], + // OpenCode regenerates config (including MCP credentials and npm + // executable symlinks) and cache on launch. Its resumable session + // database lives in data, which must remain in the checkpoint. + excludeEntries: ["config", "cache"].map( + (entry) => `${opencodeRuntimeSessionDirectoryName(nativeSessionKey(execution))}/${entry}`, + ), } : execution.provider.kind === "acpx" ? { @@ -9114,7 +9120,7 @@ export async function stageRemoteRunnerDirectory(input: { const archive = execFileSync( "tar", [...excludeArgs, "-czf", "-", "-C", input.sourcePath, "."], - { encoding: "buffer", maxBuffer: 64 * 1024 * 1024 }, + { encoding: "buffer", maxBuffer: 64 * 1024 * 1024, env: { ...process.env, COPYFILE_DISABLE: "1" } }, ); execFileSync("tar", ["-xzf", "-", "-C", stagingRoot], { input: archive, @@ -9143,11 +9149,18 @@ export async function stageRemoteRunnerDirectory(input: { const archive = execFileSync( "tar", [...excludeArgs, "-czf", "-", "-C", input.sourcePath, "."], - { encoding: "buffer", maxBuffer: 64 * 1024 * 1024 }, + { encoding: "buffer", maxBuffer: 64 * 1024 * 1024, env: { ...process.env, COPYFILE_DISABLE: "1" } }, ); const escapedTarget = input.targetPath.replaceAll("'", "'\\''"); + // Runtime instruction/skill snapshots are read-only after staging. A durable + // VM reuses this path on the next turn: permit replacing its entries without + // following symlinks into another workspace, then restore archive modes. + const prepareReadonly = input.mode === 0o555 + ? `test ! -L '${escapedTarget}' && find '${escapedTarget}' -type d -exec chmod u+w {} + && ` + : ""; const script = `umask 077; mkdir -p '${escapedTarget}' && ` + + prepareReadonly + `base64 -d | tar -xzf - -C '${escapedTarget}' && ` + `chmod ${input.mode.toString(8)} '${escapedTarget}'`; const result = await input.runner.execute({ diff --git a/server/src/vendor/paperclip-runner/index.ts b/server/src/vendor/paperclip-runner/index.ts index 37ae3f52c29..8757e840a60 100644 --- a/server/src/vendor/paperclip-runner/index.ts +++ b/server/src/vendor/paperclip-runner/index.ts @@ -83,6 +83,7 @@ export const NATIVE_RUNTIME_ASSET_SCHEMA = runner.NATIVE_RUNTIME_ASSET_SCHEMA; export const PAPERCLIP_EXECUTION_PROMPT = runner.PAPERCLIP_EXECUTION_PROMPT; export const PAPERCLIP_EXECUTION_PROMPT_REVISION = runner.PAPERCLIP_EXECUTION_PROMPT_REVISION; +export const opencodeRuntimeSessionDirectoryName = runner.opencodeRuntimeSessionDirectoryName; export const acpxRuntimeSessionDirectoryName = runner.acpxRuntimeSessionDirectoryName; export const canonicalNativeRuntimeContextDigest =