From d172197117a14b80a1eb2d2835a0e7cce2679656 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 29 Sep 2026 07:59:11 -0500 Subject: [PATCH] feat(storage): add plain directory sync with conflict preflight (#14416) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Agent runs use workspace transport to restore and collect files. > - Some directories belong to the agent across tasks. > - Those directories need plain file transport without task Git state. > - A concurrent file edit must be detected before a merge changes any file. > - This pull request adds optional plain-directory sync and conflict preflight. > - Existing task workspace sync keeps its defaults. ## Linked Issues or Issue Description Refs #14325. This is the transport prerequisite for a replacement of its instruction revision design with current agent files. ## What Changed - Add an opt-in plain-directory transport mode to command and sandbox runtimes. - Add strict merge preflight for file edits, deletions, and directory changes. - Accept identical replay after an interrupted merge. Preserve competing changes. - Set the compiled OpenCode test executable to 0755, independent of the CI host’s file-creation mask. Preserve the original startup error if cleanup also fails. ## Verification - At `ced53ae532ce6966cad5a83575d83db61af98126`, all 212 targeted transport tests passed across workspace restore, remote managed runtime, SSH fixture, and execution-target sandbox suites. Adapter-utils typecheck passed. - Real isolated SSH retry fixture previously passed with `PAPERCLIP_ENABLE_DARWIN_SSH_ENV_LAB=1`; stale deleted files remain absent while gitignored binary bytes survive. - Dependent PR #14420 passed real native local, legacy local, and native Daytona persistence E2E at `169fab46d5af21caa2269b4c1b29b69c933a6951`, which includes all production transport changes through `ced53ae53`; the subsequent two commits only fix the OpenCode test fixture. Nine tasks, three server restarts, and all cleanup checks passed. - A hosted OpenCode fixture failed twice at `ced53ae53`. Reproduced the failure locally and in Linux with `umask 0002`: the compiler created a group-writable executable, correctly rejected by the qualified launch boundary. Explicit 0755 permissions fix the test without weakening the production guard. The focused test and non-root Linux reproduction now pass under that same mask. - Before rebase, head `69e97de0475d34aac5d532e559a405eaf015fd2b` includes the deterministic fixture permission fix and preserves original bootstrap diagnostics. All production transport code is unchanged since the 212-test validation. Fresh Greptile review is 5/5 on this exact head with no unresolved findings. All 54 current-head checks passed, with two conditional skips. The full CI run completed successfully, including the previously failing OpenCode runner shard. - Merge validation on rebased head `c509d79dd190c5cb00dc65edfde209097ff21465`: all five commits are patch-identical to the reviewed branch. All 54 checks passed with two conditional skips, and fresh Greptile review is 5/5 with no findings. One retry cleared an npm archive 404 and a Cursor fixture timeout. ## Risks - New behavior is opt-in. Existing task snapshot behavior retains its defaults. - Generic strict merge preflight remains opt-in. The dependent agent-folder feature rebases changed paths before applying them to provide per-file last-sync-wins; it does not create a conflict-review queue. - This change adds no database migration, dependency, or UI. ## Model Used OpenAI Codex, GPT-6 family. The session does not expose a more specific model ID or context-window size. Reasoning, code execution, and tool use assisted this change. Live provider validation used `gpt-5.6-sol`. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../src/command-managed-runtime.ts | 4 + .../adapter-utils/src/execution-target.ts | 5 ++ .../src/remote-managed-runtime.test.ts | 27 +++++++ .../src/remote-managed-runtime.ts | 6 +- .../src/sandbox-managed-runtime.ts | 14 ++-- .../adapter-utils/src/ssh-fixture.test.ts | 22 ++++++ packages/adapter-utils/src/ssh.ts | 6 +- .../src/workspace-restore-merge.test.ts | 53 +++++++++++++ .../src/workspace-restore-merge.ts | 78 +++++++++++++++++-- .../src/live/runnerd-codex-transport.test.ts | 18 ++++- 10 files changed, 214 insertions(+), 19 deletions(-) diff --git a/packages/adapter-utils/src/command-managed-runtime.ts b/packages/adapter-utils/src/command-managed-runtime.ts index befdccf674..9abfd933c9 100644 --- a/packages/adapter-utils/src/command-managed-runtime.ts +++ b/packages/adapter-utils/src/command-managed-runtime.ts @@ -544,6 +544,8 @@ export async function prepareCommandManagedRuntime(input: { workspaceBaseline?: DirectorySnapshot; workspaceGitSnapshot?: GitWorkspaceSnapshot | null; workspaceExclude?: string[]; + /** Plain persistent directories include all files, independent of Git and task cache exclusions. */ + workspaceFileMode?: "all"; preserveAbsentOnRestore?: string[]; assets?: CommandManagedRuntimeAsset[]; /** Referenced (additional) projects to stage into the sandbox as plain, read-only trees. */ @@ -609,6 +611,7 @@ export async function prepareCommandManagedRuntime(input: { workspaceBaseline: input.workspaceBaseline, workspaceGitSnapshot: input.workspaceGitSnapshot, workspaceExclude: mergeRuntimeExcludes(input.workspaceExclude), + workspaceFileMode: input.workspaceFileMode, preserveAbsentOnRestore: input.preserveAbsentOnRestore, assets: input.assets, additionalSources: input.additionalSources, @@ -652,6 +655,7 @@ export async function prepareCommandManagedRuntime(input: { workspaceBaseline: input.workspaceBaseline, workspaceGitSnapshot: input.workspaceGitSnapshot, workspaceExclude: mergeRuntimeExcludes(input.workspaceExclude), + workspaceFileMode: input.workspaceFileMode, preserveAbsentOnRestore: input.preserveAbsentOnRestore, assets: input.assets, additionalSources: input.additionalSources, diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index ca816ca25c..6b118011d4 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -1437,6 +1437,8 @@ export async function prepareAdapterExecutionTargetRuntime(input: { workspaceBaseline?: DirectorySnapshot; workspaceGitSnapshot?: GitWorkspaceSnapshot | null; workspaceExclude?: string[]; + /** Plain persistent directories include all files, independent of Git and task cache exclusions. */ + workspaceFileMode?: "all"; preserveAbsentOnRestore?: string[]; assets?: AdapterManagedRuntimeAsset[]; /** Referenced (additional) projects to stage into the sandbox as plain, read-only trees. */ @@ -1478,6 +1480,8 @@ export async function prepareAdapterExecutionTargetRuntime(input: { workspaceLocalDir: input.workspaceLocalDir, workspaceRemoteDir: input.workspaceRemoteDir, syncWorkspace: input.syncWorkspace, + workspaceFileMode: input.workspaceFileMode, + workspaceExclude: input.workspaceExclude, assets: input.assets, additionalSources: input.additionalSources, onProgress: input.onProgress, @@ -1517,6 +1521,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: { workspaceBaseline: input.workspaceBaseline, workspaceGitSnapshot: input.workspaceGitSnapshot, workspaceExclude: input.workspaceExclude, + workspaceFileMode: input.workspaceFileMode, preserveAbsentOnRestore: input.preserveAbsentOnRestore, assets: input.assets, additionalSources: input.additionalSources, diff --git a/packages/adapter-utils/src/remote-managed-runtime.test.ts b/packages/adapter-utils/src/remote-managed-runtime.test.ts index 89659bb643..f5cb64b9d0 100644 --- a/packages/adapter-utils/src/remote-managed-runtime.test.ts +++ b/packages/adapter-utils/src/remote-managed-runtime.test.ts @@ -32,6 +32,33 @@ import { setExpensiveWorkspaceGitExecutor } from "./git-workspace-sync.js"; describe("remote managed runtime", () => { const cleanupDirs: string[] = []; + it("stages all files over SSH without Git or cache exclusions and restores the same baseline", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-ssh-plain-")); + cleanupDirs.push(root); + await mkdir(path.join(root, "node_modules")); + await writeFile(path.join(root, ".gitignore"), "node_modules/\n"); + await writeFile(path.join(root, "node_modules", "personal.bin"), Buffer.from([0, 255, 1])); + const prepared = await prepareRemoteManagedRuntime({ + spec: { host: "127.0.0.1", port: 2222, username: "fixture", remoteWorkspacePath: "/app", remoteCwd: "/app", + privateKey: "PRIVATE KEY", knownHosts: "KNOWN HOSTS", strictHostKeyChecking: true }, + runId: "plain", adapterKey: "test", workspaceLocalDir: root, + workspaceFileMode: "all", workspaceExclude: ["explicitly-excluded"], + }); + expect(prepareWorkspaceForSshExecution).toHaveBeenCalledWith(expect.objectContaining({ + localDir: root, remoteDir: prepared.workspaceRemoteDir, workspaceFileMode: "all", workspaceExclude: ["explicitly-excluded"], + })); + await prepared.restoreWorkspace(); + expect(restoreWorkspaceFromSshExecution).toHaveBeenCalledWith(expect.objectContaining({ + restoreGitHistory: false, baselineSnapshot: expect.objectContaining({ + exclude: [".paperclip-runtime", "explicitly-excluded"], + entries: expect.any(Map), + }), + })); + const args = vi.mocked(restoreWorkspaceFromSshExecution).mock.calls[0] as unknown as [{ baselineSnapshot: { entries: Map } }]; + expect(args[0].baselineSnapshot.entries.has("node_modules/personal.bin")).toBe(true); + }); + + afterEach(async () => { vi.clearAllMocks(); while (cleanupDirs.length > 0) { diff --git a/packages/adapter-utils/src/remote-managed-runtime.ts b/packages/adapter-utils/src/remote-managed-runtime.ts index ff394dc7fb..cb47f3caed 100644 --- a/packages/adapter-utils/src/remote-managed-runtime.ts +++ b/packages/adapter-utils/src/remote-managed-runtime.ts @@ -113,6 +113,8 @@ export async function prepareRemoteManagedRuntime(input: { workspaceLocalDir: string; workspaceRemoteDir?: string; syncWorkspace?: boolean; + workspaceFileMode?: "all"; + workspaceExclude?: string[]; assets?: RemoteManagedRuntimeAsset[]; /** Referenced (additional) projects to stage as plain, read-only trees. */ additionalSources?: SandboxAdditionalSource[]; @@ -139,13 +141,15 @@ export async function prepareRemoteManagedRuntime(input: { localDir: input.workspaceLocalDir, remoteDir: workspaceRemoteDir, onProgress: input.onProgress, + workspaceFileMode: input.workspaceFileMode, + workspaceExclude: input.workspaceExclude, }) : null; const baselineSnapshot = preparedWorkspace ? await captureDirectorySnapshot(input.workspaceLocalDir, { exclude: preparedWorkspace.gitBacked ? [...GIT_ARCHIVE_EXCLUDES, ".paperclip-runtime"] - : [".paperclip-runtime"], + : [".paperclip-runtime", ...(input.workspaceFileMode === "all" ? input.workspaceExclude ?? [] : [])], }) : null; diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.ts b/packages/adapter-utils/src/sandbox-managed-runtime.ts index 3f83e61639..10eeab606e 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.ts @@ -1102,6 +1102,8 @@ export async function prepareSandboxManagedRuntime(input: { workspaceBaseline?: DirectorySnapshot; workspaceGitSnapshot?: GitWorkspaceSnapshot | null; workspaceExclude?: string[]; + /** Plain persistent directories include all files, independent of Git and task cache exclusions. */ + workspaceFileMode?: "all"; preserveAbsentOnRestore?: string[]; assets?: SandboxManagedRuntimeAsset[]; /** @@ -1179,7 +1181,7 @@ export async function prepareSandboxManagedRuntime(input: { // The git enumeration (`git status --ignored`, the HEAD diffs, `ls-files`). // It reads git's own bookkeeping to decide what to include/exclude, so it is // usually fast, but on a large working tree the `--ignored` walk is not free. - const gitSnapshot = syncWorkspace + const gitSnapshot = syncWorkspace && input.workspaceFileMode !== "all" ? input.workspaceGitSnapshot !== undefined ? input.workspaceGitSnapshot : await runStepSpan("snapshot.git", () => @@ -1195,7 +1197,7 @@ export async function prepareSandboxManagedRuntime(input: { // A selected subfolder has no cloneable Git snapshot, but its parent // repository's ignore rules still govern which files may leave the host. // Use the same bounded, path-relative resolver as referenced project trees. - const directoryIgnore = syncWorkspace && !gitSnapshot + const directoryIgnore = syncWorkspace && !gitSnapshot && input.workspaceFileMode !== "all" ? await resolveReferencedSourceIgnore(input.workspaceLocalDir) : null; if (directoryIgnore?.kind === "failed") { @@ -1203,14 +1205,14 @@ export async function prepareSandboxManagedRuntime(input: { } const gitIgnoredExcludes = directoryIgnore?.kind === "git" ? directoryIgnore.ignoredPaths : undefined; const workspaceArchiveExclude = mergeExcludes( - SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES, - [...GIT_ARCHIVE_EXCLUDES], + input.workspaceFileMode === "all" ? [] : SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES, + input.workspaceFileMode === "all" ? [] : [...GIT_ARCHIVE_EXCLUDES], input.workspaceExclude, gitIgnoredExcludes, ); const restoreExclude = mergeExcludes( - SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES, - [...GIT_ARCHIVE_EXCLUDES], + input.workspaceFileMode === "all" ? [] : SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES, + input.workspaceFileMode === "all" ? [] : [...GIT_ARCHIVE_EXCLUDES], [".paperclip-runtime"], input.preserveAbsentOnRestore, input.workspaceExclude, diff --git a/packages/adapter-utils/src/ssh-fixture.test.ts b/packages/adapter-utils/src/ssh-fixture.test.ts index c9c7657f56..83c4ad7e6d 100644 --- a/packages/adapter-utils/src/ssh-fixture.test.ts +++ b/packages/adapter-utils/src/ssh-fixture.test.ts @@ -712,6 +712,28 @@ describe("ssh env-lab fixture", () => { expect(result.stdout).toContain("{\"token\":\"secret\"}"); }, SSH_FIXTURE_TEST_TIMEOUT_MS); + it("clears stale files when plain SSH preparation is retried", async () => { + const rootDir = await createFixtureRootDir(); + const localDir = path.join(rootDir, "plain-local"); + await mkdir(path.join(localDir, "node_modules"), { recursive: true }); + await git(localDir, ["init"]); + await writeFile(path.join(localDir, ".gitignore"), "node_modules/\n"); + await writeFile(path.join(localDir, "removed.txt"), "remove on retry"); + const binary = Buffer.from([0, 255, 1]); + await writeFile(path.join(localDir, "node_modules", "personal.bin"), binary); + const started = await startSshEnvLabFixtureOrSkip(path.join(rootDir, "state.json"), "SSH plain retry"); + if (!started) return; + const config = await buildSshEnvLabFixtureConfig(started); + const input = { spec: { ...config, remoteCwd: started.workspaceDir }, localDir, + remoteDir: started.workspaceDir, workspaceFileMode: "all" as const }; + expect(await prepareWorkspaceForSshExecution(input)).toEqual({ gitBacked: false }); + expect(await readFile(path.join(started.workspaceDir, "node_modules", "personal.bin"))).toEqual(binary); + await rm(path.join(localDir, "removed.txt")); + await prepareWorkspaceForSshExecution(input); + await expect(stat(path.join(started.workspaceDir, "removed.txt"))).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readFile(path.join(started.workspaceDir, "node_modules", "personal.bin"))).toEqual(binary); + }, SSH_FIXTURE_TEST_TIMEOUT_MS); + it("round-trips a git workspace through the SSH fixture", async () => { const rootDir = await createFixtureRootDir(); const statePath = path.join(rootDir, "state.json"); diff --git a/packages/adapter-utils/src/ssh.ts b/packages/adapter-utils/src/ssh.ts index 084bb0803d..89fd456c5d 100644 --- a/packages/adapter-utils/src/ssh.ts +++ b/packages/adapter-utils/src/ssh.ts @@ -1558,9 +1558,11 @@ export async function prepareWorkspaceForSshExecution(input: { localDir: string; remoteDir?: string; onProgress?: RuntimeProgressSink; + workspaceFileMode?: "all"; + workspaceExclude?: string[]; }): Promise<{ gitBacked: boolean }> { const remoteDir = input.remoteDir ?? input.spec.remoteCwd; - const gitSnapshot = await readLocalGitWorkspaceSnapshot(input.localDir); + const gitSnapshot = input.workspaceFileMode === "all" ? null : await readLocalGitWorkspaceSnapshot(input.localDir); if (gitSnapshot) { await importGitWorkspaceToSsh({ @@ -1595,7 +1597,7 @@ export async function prepareWorkspaceForSshExecution(input: { spec: input.spec, localDir: input.localDir, remoteDir, - exclude: [".paperclip-runtime"], + exclude: [".paperclip-runtime", ...(input.workspaceFileMode === "all" ? input.workspaceExclude ?? [] : [])], onProgress: input.onProgress, progressLabel: "workspace", }); diff --git a/packages/adapter-utils/src/workspace-restore-merge.test.ts b/packages/adapter-utils/src/workspace-restore-merge.test.ts index 7833997dd4..914d99ac75 100644 --- a/packages/adapter-utils/src/workspace-restore-merge.test.ts +++ b/packages/adapter-utils/src/workspace-restore-merge.test.ts @@ -605,3 +605,56 @@ describe("workspace restore merge", () => { }); }); }); + +describe("conflict-preserving directory restore", () => { + it("preflights competing edits before applying any other change and deduplicates replay", async () => { + const root = await fsPromises.realpath(await mkdtemp(path.join(os.tmpdir(), "directory-cas-"))); + const source = path.join(root, "source"), target = path.join(root, "target"); + try { + await mkdir(target); + await writeFile(path.join(target, "conflict"), "baseline"); + const baseline = await captureDirectorySnapshot(target); + await fsPromises.cp(target, source, { recursive: true }); + await writeFile(path.join(source, "conflict"), "incoming"); + await writeFile(path.join(source, "independent"), "also incoming"); + await writeFile(path.join(target, "conflict"), "board"); + await expect(mergeDirectoryWithBaseline({ baseline, sourceDir: source, targetDir: target, conflictPolicy: "reject" })).rejects.toMatchObject({ code: "DIRECTORY_MERGE_CONFLICT", paths: ["conflict"] }); + await expect(stat(path.join(target, "independent"))).rejects.toMatchObject({ code: "ENOENT" }); + await writeFile(path.join(target, "conflict"), "baseline"); + await expect(mergeDirectoryWithBaseline({ baseline, sourceDir: source, targetDir: target, conflictPolicy: "reject", afterApply: async () => { throw new Error("receipt interrupted"); } })).rejects.toThrow("receipt interrupted"); + await mergeDirectoryWithBaseline({ baseline, sourceDir: source, targetDir: target, conflictPolicy: "reject" }); + expect(await readFile(path.join(target, "independent"), "utf8")).toBe("also incoming"); + } finally { await rm(root, { recursive: true, force: true }); } + }); + + it("preserves a newly added child when another run removes or replaces its parent", async () => { + const root = await fsPromises.realpath(await mkdtemp(path.join(os.tmpdir(), "directory-delete-cas-"))); + const source = path.join(root, "source"), target = path.join(root, "target"); + try { + await mkdir(path.join(target, "folder"), { recursive: true }); + const baseline = await captureDirectorySnapshot(target); + await mkdir(source); + await writeFile(path.join(target, "folder", "new"), "concurrent"); + await expect(mergeDirectoryWithBaseline({ baseline, sourceDir: source, targetDir: target, conflictPolicy: "reject" })).rejects.toMatchObject({ paths: ["folder/new"] }); + expect(await readFile(path.join(target, "folder", "new"), "utf8")).toBe("concurrent"); + } finally { await rm(root, { recursive: true, force: true }); } + }); +}); + + +it("strict preflight preserves excluded descendants when a directory becomes a file", async () => { + const root = await fsPromises.realpath(await mkdtemp(path.join(os.tmpdir(), "directory-excluded-cas-"))); + const target = path.join(root, "target"), source = path.join(root, "source"); + await mkdir(path.join(target, "folder", "node_modules"), { recursive: true }); + await writeFile(path.join(target, "folder", "node_modules", "keep"), "excluded contents"); + const baseline = await captureDirectorySnapshot(target, { exclude: ["*/node_modules"], diskBacked: true }); + try { + await mkdir(source); + await writeFile(path.join(source, "folder"), "replacement"); + await writeFile(path.join(source, "independent"), "must not partially apply"); + await expect(mergeDirectoryWithBaseline({ baseline, sourceDir: source, targetDir: target, conflictPolicy: "reject" })) + .rejects.toMatchObject({ code: "DIRECTORY_MERGE_CONFLICT", paths: expect.arrayContaining(["folder/node_modules/keep"]) }); + expect(await readFile(path.join(target, "folder", "node_modules", "keep"), "utf8")).toBe("excluded contents"); + await expect(stat(path.join(target, "independent"))).rejects.toMatchObject({ code: "ENOENT" }); + } finally { await disposeDirectorySnapshot(baseline); await rm(root, { recursive: true, force: true }); } +}); diff --git a/packages/adapter-utils/src/workspace-restore-merge.ts b/packages/adapter-utils/src/workspace-restore-merge.ts index 88e89e8f92..60e3ce539f 100644 --- a/packages/adapter-utils/src/workspace-restore-merge.ts +++ b/packages/adapter-utils/src/workspace-restore-merge.ts @@ -1,4 +1,4 @@ -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { createReadStream } from "node:fs"; import { constants as fsConstants, promises as fs } from "node:fs"; import path from "node:path"; @@ -442,16 +442,26 @@ async function copySnapshotEntry(sourceDir: string, targetDir: string, relative: } await fs.mkdir(path.dirname(targetPath), { recursive: true }); - await fs.rm(targetPath, { recursive: true, force: true }).catch(() => undefined); if (entry.kind === "symlink") { + await fs.rm(targetPath, { recursive: true, force: true }); await fs.symlink(entry.target, targetPath); return; } + // An interrupted restore must not leave a truncated current file. Keep the + // incoming tree until its owner records success; exact retries deduplicate. + const temporary = path.join(path.dirname(targetPath), `.paperclip-merge-${randomUUID()}`); + try { + await fs.copyFile(sourcePath, temporary, fsConstants.COPYFILE_FICLONE).catch(async () => { + await fs.copyFile(sourcePath, temporary); + }); + await fs.chmod(temporary, entry.mode); + const file = await fs.open(temporary, "r"); + try { await file.sync(); } finally { await file.close(); } + const existing = await fs.lstat(targetPath).catch(() => null); + if (existing?.isDirectory()) await fs.rm(targetPath, { recursive: true, force: true }); + await fs.rename(temporary, targetPath); + } finally { await fs.rm(temporary, { force: true }); } - await fs.copyFile(sourcePath, targetPath, fsConstants.COPYFILE_FICLONE).catch(async () => { - await fs.copyFile(sourcePath, targetPath); - }); - await fs.chmod(targetPath, entry.mode); } export async function captureDirectorySnapshot( @@ -511,10 +521,54 @@ function orderedEntries(snapshot: DirectorySnapshot, reverse = false): Iterable< return [...snapshot.entries].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0) * (reverse ? -1 : 1)); } +export class DirectoryMergeConflict extends Error { + readonly code = "DIRECTORY_MERGE_CONFLICT"; + constructor(readonly paths: string[]) { + super("Directory contents changed concurrently"); + } +} + +/** Preflight the entire delta before writing. Identical replays are safe after + * an interrupted apply; unrelated edits are left alone. No history is retained. */ +export function directoryMergeConflicts(baseline: DirectorySnapshot, source: DirectorySnapshot, current: DirectorySnapshot): string[] { + const same = (a: SnapshotEntry | undefined, b: SnapshotEntry | undefined) => + (!a && !b) || entriesMatch(a, b); + const conflicts = new Set(); + function* changedPaths() { + for (const [name] of baseline.entries) yield name; + for (const [name] of source.entries) if (!baseline.entries.has(name)) yield name; + } + for (const relative of changedPaths()) { + const before = baseline.entries.get(relative); + const incoming = source.entries.get(relative); + const present = current.entries.get(relative); + if (same(before, incoming) || same(incoming, present)) continue; + if (!same(before, present)) conflicts.add(relative); + // A parent removed/replaced by another writer must never be traversed. + for (let parent = path.posix.dirname(relative); parent !== "."; parent = path.posix.dirname(parent)) { + if (current.entries.get(parent)?.kind !== "dir" && + !same(current.entries.get(parent), baseline.entries.get(parent))) conflicts.add(parent); + } + } + // Stream each current entry once. A replacement must not remove children + // omitted from the baseline, including excluded or newly created files. + for (const [child, entry] of current.entries) { + if (same(entry, baseline.entries.get(child)) || same(entry, source.entries.get(child))) continue; + for (let parent = path.posix.dirname(child); parent !== "."; parent = path.posix.dirname(parent)) { + if (baseline.entries.get(parent)?.kind === "dir" && source.entries.get(parent)?.kind !== "dir") { + conflicts.add(child); + break; + } + } + } + return [...conflicts].sort(); +} + export async function mergeDirectoryWithBaseline(input: { baseline: DirectorySnapshot; sourceDir: string; targetDir: string; + conflictPolicy?: "reject"; beforeApply?: () => Promise; afterApply?: () => Promise; }): Promise { @@ -523,8 +577,15 @@ export async function mergeDirectoryWithBaseline(input: { try { await withDirectoryMergeLock(input.targetDir, async (canonicalTargetDir) => { await input.beforeApply?.(); - const current = await captureDirectorySnapshot(canonicalTargetDir, options); + // Strict preflight must see excluded children before a directory is + // replaced. The merge still applies only the filtered source/baseline. + const current = await captureDirectorySnapshot(canonicalTargetDir, + input.conflictPolicy === "reject" ? { exclude: [], diskBacked: true } : options); try { + if (input.conflictPolicy === "reject") { + const conflicts = directoryMergeConflicts(input.baseline, source, current); + if (conflicts.length) throw new DirectoryMergeConflict(conflicts); + } for (const [relative, baselineEntry] of orderedEntries(input.baseline)) { if (baselineEntry.kind === "dir" || source.entries.has(relative)) continue; if (!entriesMatch(current.entries.get(relative), baselineEntry)) continue; @@ -537,7 +598,8 @@ export async function mergeDirectoryWithBaseline(input: { }); } for (const [relative, entry] of orderedEntries(source)) { - if (!entriesMatch(input.baseline.entries.get(relative), entry)) await copySnapshotEntry(input.sourceDir, canonicalTargetDir, relative, entry); + if (!entriesMatch(input.baseline.entries.get(relative), entry) && + !(input.conflictPolicy === "reject" && entriesMatch(current.entries.get(relative), entry))) await copySnapshotEntry(input.sourceDir, canonicalTargetDir, relative, entry); } await input.afterApply?.(); } finally { await disposeDirectorySnapshot(current); } diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 691dda544f..1baae36920 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -1,4 +1,5 @@ import { + chmod, cp, mkdir, lstat, @@ -7273,6 +7274,8 @@ it("preserves prepared input through runnerd and the real OpenCode proxy boundar execFileSync("cc", ["-x", "c", "-o", executable, "-"], { input: `#include \n#include \nint main(int argc, char **argv) { char **args = calloc(argc + 2, sizeof(char *)); args[0] = ${JSON.stringify(process.execPath)}; args[1] = ${JSON.stringify(fixture)}; for (int i = 1; i < argc; i++) args[i + 1] = argv[i]; execv(args[0], args); return 127; }`, }); + // CI may use umask 0002; qualified executables cannot be group-writable. + await chmod(executable, 0o755); // Use the production bundler without depending on (or mutating) shared dist // artifacts. The Vitest CI lane builds Rust but does not build TypeScript. const proxy = join(root, "opencode-app-server-proxy.cjs"); @@ -7316,6 +7319,7 @@ it("preserves prepared input through runnerd and the real OpenCode proxy boundar task: { prompt: "Keep this request unchanged." }, completionContract: { revision: "prepared-v1", criteria: task.completionContract.criteria }, }); + let failure: unknown; try { session = await driver.openSession({ runId: "prepared-opencode", normalizedSessionId: "prepared-opencode", workingDirectory: root }); await session.startTurn({ message: { role: "user", text: prepared } }); @@ -7326,9 +7330,19 @@ it("preserves prepared input through runnerd and the real OpenCode proxy boundar expect(sessionRoots).toHaveLength(1); const requests = (await readFile(join(runtime, sessionRoots[0]!.name, "data/fake-prompt-requests.ndjson"), "utf8")).trim().split("\n").map((line) => JSON.parse(line)); expect(requests.map((request) => request.parts)).toEqual([[{ type: "text", text: prepared }]]); + } catch (error) { + failure = error; } finally { - await session?.close(); - await bundle.transport.close(); + try { + await session?.close(); + await bundle.transport.close(); + } catch (error) { + // Preserve bootstrap failures when cleanup independently cannot suspend. + failure ??= error; + } await rm(root, { recursive: true, force: true }); } + if (failure) { + throw new Error(`${String(failure)}\n${bundle.evidence().diagnostics.join("\n")}`, { cause: failure }); + } }, 30_000);