diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts index 6e309cc09c7..4233e5ce594 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts @@ -2476,12 +2476,16 @@ function sessionRoot( runtimeDirectory: string, normalizedSessionId: string, ): string { + return join(resolve(runtimeDirectory), opencodeRuntimeSessionDirectoryName(normalizedSessionId)); +} + +export function opencodeRuntimeSessionDirectoryName(normalizedSessionId: string): string { const safe = normalizedSessionId .replace(/[^a-zA-Z0-9._-]/g, "_") .slice(0, 120); if (!safe || safe === "." || safe === "..") throw new Error("Invalid normalized OpenCode session id"); - return join(resolve(runtimeDirectory), safe); + return safe; } function validateWorkspace(value: string): string { diff --git a/packages/paperclip-runner/src/index.ts b/packages/paperclip-runner/src/index.ts index 7aeec692b85..ef9130167e1 100644 --- a/packages/paperclip-runner/src/index.ts +++ b/packages/paperclip-runner/src/index.ts @@ -18,6 +18,7 @@ export { } from "./backends/native-backend-factory.js"; export { OpenCodeServerDriver, + opencodeRuntimeSessionDirectoryName, type OpenCodeServerDriverOptions, } from "./drivers/opencode/opencode-server-driver.js"; export { diff --git a/packages/plugins/sandbox-providers/exe-dev/README.md b/packages/plugins/sandbox-providers/exe-dev/README.md index 651c0f57599..8622db19593 100644 --- a/packages/plugins/sandbox-providers/exe-dev/README.md +++ b/packages/plugins/sandbox-providers/exe-dev/README.md @@ -81,6 +81,9 @@ but a changed remote file takes precedence when the same file also changed on th host. Use isolated worktrees for concurrent editing; this is not a conflict merge. Native runs additionally use the existing verified harness/session checkpoint and restart-recovery contract. +OpenCode checkpoints preserve its session database and history while excluding +regenerated configuration (including launch credentials and npm executable +aliases) and caches. Codex launch credentials and scratch aliases are also excluded. The existing exclusions still apply, including dependencies, generated/cache folders, ignored files, and runtime scratch. diff --git a/server/src/__tests__/exe-environment-resource.test.ts b/server/src/__tests__/exe-environment-resource.test.ts index f448b67620a..ba8f618372b 100644 --- a/server/src/__tests__/exe-environment-resource.test.ts +++ b/server/src/__tests__/exe-environment-resource.test.ts @@ -37,6 +37,19 @@ const support = await getEmbeddedPostgresTestSupport(); await Promise.all(Array.from({ length: 8 }, () => bindEnvironmentResource(db, id, companyId, binding))); expect(await readEnvironmentResourceBinding(db, id, companyId)).toEqual(binding); }); + it("keeps the configuration revision stable while attesting VM identity", async () => { + const { id, companyId, binding } = await fixture(); + const revision = new Date("2020-01-01T00:00:00Z"); + await db.update(environments).set({ updatedAt: revision }).where(eq(environments.id, id)); + await bindEnvironmentResource(db, id, companyId, binding); + await bindEnvironmentResource(db, id, companyId, binding); + const [bound] = await db.select().from(environments).where(eq(environments.id, id)); + expect(bound.updatedAt).toEqual(revision); + expect(await readEnvironmentResourceBinding(db, id, companyId)).toEqual(binding); + await environmentService(db).update(id, { config: { provider: "exe-dev", vmName: "operator-change" } }); + const [edited] = await db.select().from(environments).where(eq(environments.id, id)); + expect(edited.updatedAt.getTime()).toBeGreaterThan(revision.getTime()); + }); it("allows only one identity when competing first acquisitions disagree", async () => { const { id, companyId, binding } = await fixture(); const results = await Promise.allSettled([ diff --git a/server/src/__tests__/native-finalization-recovery.test.ts b/server/src/__tests__/native-finalization-recovery.test.ts index 90d0be71741..bfd3f993c21 100644 --- a/server/src/__tests__/native-finalization-recovery.test.ts +++ b/server/src/__tests__/native-finalization-recovery.test.ts @@ -213,6 +213,17 @@ describe("P6-16/P6-25/P6-28 native finalization recovery", () => { await temporary.cleanup(); }); + it("leaves finalization to an active heartbeat instead of racing its copyback", async () => { + const [before] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + const isRunActive = vi.fn((candidate: string) => candidate === runId); + await expect(reconcileNativeFinalizations(db, [runId], { isRunActive })).resolves.toEqual([]); + expect(isRunActive).toHaveBeenCalledWith(runId); + const [after] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + expect(after).toEqual(before); + // The following recovery tests omit the live owner and must still settle + // these persisted runs, as a new controller does after restart. + }); + it("fails closed into bounded named recovery without consulting the live flag or falling back", async () => { await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([ expect.objectContaining({ phase: "retryable_failure", failureCode: "native_finalization_invalid" }), diff --git a/server/src/services/environment-resource-binding.ts b/server/src/services/environment-resource-binding.ts index 3767691bf1f..3ffc9ccb546 100644 --- a/server/src/services/environment-resource-binding.ts +++ b/server/src/services/environment-resource-binding.ts @@ -28,7 +28,9 @@ export async function bindEnvironmentResource(db: Db, environmentId: string, com const encoded = JSON.stringify(binding); const rows = await db.update(environments).set({ metadata: sql`coalesce(${environments.metadata}, '{}'::jsonb) || jsonb_build_object('environmentResourceBinding', ${encoded}::jsonb)`, - updatedAt: new Date(), + // Binding attestation is runtime state. updatedAt is also the configuration + // revision used for session compatibility; changing it would rotate every + // agent session when this VM is first bound or another lease is acquired. }).where(and(eq(environments.id, environmentId), sql`(${environments.metadata}->'environmentResourceBinding' IS NULL OR ${environments.metadata}->'environmentResourceBinding' = ${encoded}::jsonb)`)).returning({ id: environments.id }); if (rows.length !== 1) throw conflict("Durable VM identity changed; explicit recovery to a new environment is required"); } diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index abb5961d5f2..0b81db43b3a 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -14758,6 +14758,7 @@ export function heartbeatService( // whether any provider authority needs to be reopened. await reconcileNativeFinalizations(db, undefined, { environmentRuntime, + isRunActive: (runId) => activeRunExecutions.has(runId), onWorkspaceSettled: settleRecoveredNativeWorkspace, }); scheduleRetainedNativeSessionCleanup(); @@ -18627,6 +18628,7 @@ export function heartbeatService( // mode, never the current feature flag. await reconcileNativeFinalizations(db, undefined, { environmentRuntime, + isRunActive: (runId) => activeRunExecutions.has(runId), onWorkspaceSettled: settleRecoveredNativeWorkspace, }).catch((error) => { logger.warn( diff --git a/server/src/services/native-runtime/native-finalization-reconciler.ts b/server/src/services/native-runtime/native-finalization-reconciler.ts index c88479166bb..5c8344dfad7 100644 --- a/server/src/services/native-runtime/native-finalization-reconciler.ts +++ b/server/src/services/native-runtime/native-finalization-reconciler.ts @@ -538,6 +538,8 @@ export async function reconcileNativeFinalizations( runIds?: string[], options: { environmentRuntime?: EnvironmentRuntimeService; + /** The live heartbeat owns its copyback/status suffix until its finally exits. */ + isRunActive?: (runId: string) => boolean; onWorkspaceSettled?: (input: { runId: string; companyId: string; @@ -594,6 +596,10 @@ export async function reconcileNativeFinalizations( )); const results = []; for (const row of rows) { + // A provider can persist its result before the heartbeat finishes exporting + // files. Concurrent recovery would reuse and remove that same export archive. + // Startup has no live owner, so crashed runs still follow durable recovery. + if (options.isRunActive?.(row.runId)) continue; const pendingEffects = row.decisionId ? await db.select({ id: statusDecisionEffects.id }).from(statusDecisionEffects).where(and( eq(statusDecisionEffects.companyId, row.companyId), diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 4f40c3e0904..d97f7568488 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { access, + chmod, lstat, mkdir, mkdtemp, @@ -1578,6 +1579,39 @@ describe("remote provider checkpoint snapshots", () => { expect(syncOut).toHaveBeenCalledOnce(); }); + it("persists OpenCode history while omitting generated config, credentials, cache, and executable aliases", async () => { + const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-checkpoint-")); + const sourcePath = join(root, "source"); + const targetPath = join(root, "durable"); + const sessionDirectory = "session"; + try { + for (const directory of ["data", "config", "cache"]) { + await mkdir(join(sourcePath, sessionDirectory, directory), { recursive: true }); + } + await writeFile(join(sourcePath, sessionDirectory, "data", "session.db"), "resume-history"); + await writeFile(join(sourcePath, sessionDirectory, "config", "opencode.json"), "launch-secret"); + await symlink("/outside/executable", join(sourcePath, sessionDirectory, "config", "npm-bin")); + await symlink("/outside/cache", join(sourcePath, sessionDirectory, "cache", "bin")); + const profile = resolveNativeHarnessPersistenceProfile({ + provider: { kind: "opencode" }, + session: { normalizedSessionId: "session", driverKind: "opencode_server" }, + } as NativeExecutionInputV1); + const directory = profile.directories.find((entry) => entry.name === "opencode")!; + const execute = vi.fn(async ({ command, args }: { command: string; args: string[] }) => ({ + exitCode: 0, timedOut: false, stderr: "", + stdout: execFileSync(command, args, { encoding: "utf8" }), + })); + await syncRemoteRunnerDirectoryOut({ runner: { execute } as never, + sourcePath, targetPath, mode: 0o700, excludeEntries: directory.excludeEntries }); + expect(await readFile(join(targetPath, sessionDirectory, "data", "session.db"), "utf8")).toBe("resume-history"); + await expect(access(join(targetPath, sessionDirectory, "config"))).rejects.toThrow(); + await expect(access(join(targetPath, sessionDirectory, "cache"))).rejects.toThrow(); + expect(await readFile(join(sourcePath, sessionDirectory, "config", "opencode.json"), "utf8")).toBe("launch-secret"); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + it("rejects unsafe relative checkpoint exclusions", async () => { const execute = vi.fn().mockResolvedValue({ exitCode: 0, @@ -1643,6 +1677,42 @@ describe("remote provider checkpoint snapshots", () => { }); describe("remote provider checkpoint restores", () => { + it("replaces read-only runtime instructions on a reused VM without writing through file aliases", async () => { + const root = await mkdtemp(join(tmpdir(), "paperclip-readonly-restage-")); + const sourcePath = join(root, "source"); + const targetPath = join(root, "remote"); + const outside = join(root, "outside.txt"); + try { + await mkdir(join(sourcePath, "nested"), { recursive: true }); + await writeFile(join(sourcePath, "nested", "AGENTS.md"), "first"); + await chmod(join(sourcePath, "nested", "AGENTS.md"), 0o444); + await chmod(join(sourcePath, "nested"), 0o555); + const execute = vi.fn(async ({ command, args, stdin }: { command: string; args: string[]; stdin?: string }) => ({ + exitCode: 0, timedOut: false, stderr: "", + stdout: execFileSync(command, args, { input: stdin, encoding: "utf8" }), + })); + const input = { target: {} as never, runner: { execute } as never, sourcePath, targetPath, mode: 0o555 }; + await stageRemoteRunnerDirectory(input); + await chmod(join(sourcePath, "nested", "AGENTS.md"), 0o644); + await writeFile(join(sourcePath, "nested", "AGENTS.md"), "revised"); + await chmod(join(sourcePath, "nested", "AGENTS.md"), 0o444); + await stageRemoteRunnerDirectory(input); + expect(await readFile(join(targetPath, "nested", "AGENTS.md"), "utf8")).toBe("revised"); + expect((await lstat(targetPath)).mode & 0o777).toBe(0o555); + await writeFile(outside, "untouched"); + await chmod(join(targetPath, "nested"), 0o755); + await rm(join(targetPath, "nested", "AGENTS.md")); + await symlink(outside, join(targetPath, "nested", "AGENTS.md")); + await chmod(join(targetPath, "nested"), 0o555); + await stageRemoteRunnerDirectory(input); + expect(await readFile(outside, "utf8")).toBe("untouched"); + expect((await lstat(join(targetPath, "nested", "AGENTS.md"))).isSymbolicLink()).toBe(false); + } finally { + execFileSync("chmod", ["-R", "u+w", root]); + await rm(root, { recursive: true, force: true }); + } + }); + it("does not upload excluded Codex scratch trees or credentials", async () => { const sourcePath = await mkdtemp( join(tmpdir(), "paperclip-codex-restore-source-"), diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 2173f6b6f99..9ef7098e0bf 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -59,6 +59,7 @@ import { completeRetainedNativeSessionCleanup, completeTerminatedLocalNativeSessionCleanup, acpxRuntimeSessionDirectoryName, + opencodeRuntimeSessionDirectoryName, createNativeSessionBackend, createRunnerdCodexTransport, defaultCapabilityRunnerdBinary, @@ -5036,7 +5037,12 @@ export function resolveNativeHarnessPersistenceProfile( ? { name: "opencode", location: "filesystem", - excludeEntries: [], + // OpenCode regenerates config (including MCP credentials and npm + // executable symlinks) and cache on launch. Its resumable session + // database lives in data, which must remain in the checkpoint. + excludeEntries: ["config", "cache"].map( + (entry) => `${opencodeRuntimeSessionDirectoryName(nativeSessionKey(execution))}/${entry}`, + ), } : execution.provider.kind === "acpx" ? { @@ -9114,7 +9120,7 @@ export async function stageRemoteRunnerDirectory(input: { const archive = execFileSync( "tar", [...excludeArgs, "-czf", "-", "-C", input.sourcePath, "."], - { encoding: "buffer", maxBuffer: 64 * 1024 * 1024 }, + { encoding: "buffer", maxBuffer: 64 * 1024 * 1024, env: { ...process.env, COPYFILE_DISABLE: "1" } }, ); execFileSync("tar", ["-xzf", "-", "-C", stagingRoot], { input: archive, @@ -9143,11 +9149,18 @@ export async function stageRemoteRunnerDirectory(input: { const archive = execFileSync( "tar", [...excludeArgs, "-czf", "-", "-C", input.sourcePath, "."], - { encoding: "buffer", maxBuffer: 64 * 1024 * 1024 }, + { encoding: "buffer", maxBuffer: 64 * 1024 * 1024, env: { ...process.env, COPYFILE_DISABLE: "1" } }, ); const escapedTarget = input.targetPath.replaceAll("'", "'\\''"); + // Runtime instruction/skill snapshots are read-only after staging. A durable + // VM reuses this path on the next turn: permit replacing its entries without + // following symlinks into another workspace, then restore archive modes. + const prepareReadonly = input.mode === 0o555 + ? `test ! -L '${escapedTarget}' && find '${escapedTarget}' -type d -exec chmod u+w {} + && ` + : ""; const script = `umask 077; mkdir -p '${escapedTarget}' && ` + + prepareReadonly + `base64 -d | tar -xzf - -C '${escapedTarget}' && ` + `chmod ${input.mode.toString(8)} '${escapedTarget}'`; const result = await input.runner.execute({ diff --git a/server/src/vendor/paperclip-runner/index.ts b/server/src/vendor/paperclip-runner/index.ts index 37ae3f52c29..8757e840a60 100644 --- a/server/src/vendor/paperclip-runner/index.ts +++ b/server/src/vendor/paperclip-runner/index.ts @@ -83,6 +83,7 @@ export const NATIVE_RUNTIME_ASSET_SCHEMA = runner.NATIVE_RUNTIME_ASSET_SCHEMA; export const PAPERCLIP_EXECUTION_PROMPT = runner.PAPERCLIP_EXECUTION_PROMPT; export const PAPERCLIP_EXECUTION_PROMPT_REVISION = runner.PAPERCLIP_EXECUTION_PROMPT_REVISION; +export const opencodeRuntimeSessionDirectoryName = runner.opencodeRuntimeSessionDirectoryName; export const acpxRuntimeSessionDirectoryName = runner.acpxRuntimeSessionDirectoryName; export const canonicalNativeRuntimeContextDigest =