diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 4a1aea1d23c..3548914d2fc 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -780,6 +780,15 @@ conflict detection. Legacy adapter snapshots remain inline. Snapshot-only worker do not load the adapter process-launch helpers. The directory merge loads those helpers when it resolves its instance-scoped lock root. +Portable remote workspace capture stages only self-contained Git metadata. It +streams working files into the archive without a second working-tree copy and +does not repack an already dissociated mirror. Each streamed file must match its +recorded hash, size and mode and retain its filesystem identity while read. +Capture rechecks the complete source inventory and Git state before returning a +receipt. A changed source or cancelled capture removes its incomplete archive; +an existing archive remains untouched. Temporary space must still hold the +normalized Git metadata and the complete archive together. + ### Remote Codex model compatibility Fresh remote Codex Runner runs check the selected image CLI before saving the diff --git a/maps/agent-access-map/workspaces.md b/maps/agent-access-map/workspaces.md index c2b261a7f4f..cc71214ed52 100644 --- a/maps/agent-access-map/workspaces.md +++ b/maps/agent-access-map/workspaces.md @@ -74,6 +74,9 @@ without a controller checkout fallback. Current authorization and responsible-us checks still apply; read-only cross-project referenced folders and explicitly concurrent shared writers are rejected. Managed mutable same-project repositories are supported. +Portable capture streams validated working files while staging self-contained +Git metadata. The smaller temporary-space requirement grants no new agent access +or execution authority. Changed sources still fail before a save receipt. An inherited isolated workspace or operator branch can be reserved through the task's current persisted binding. The original source task remains provenance. Reservation checks task/run, binding, trust, and retained source visibility before diff --git a/maps/feature-map/workspaces.md b/maps/feature-map/workspaces.md index 83d5504ff63..bd4f98cfede 100644 --- a/maps/feature-map/workspaces.md +++ b/maps/feature-map/workspaces.md @@ -83,6 +83,14 @@ or original Git remote. Read-only cross-project referenced folders must fail admission. Separate workspaces may run concurrently; shared concurrent writers must be rejected. +Repeat remote capture with enough free space for the archive and normalized Git +metadata, but not another working-tree copy. Verify saved files and Git state +after restore. A concurrent file change must fail capture without publishing a +receipt or retaining a partial archive. The +[portable archive suite](../../server/src/services/remote-workspace-snapshot-archive.test.ts) +checks the space allowance and concurrent-change boundary; its filesystem quota +is controlled test input, not a live sandbox qualification. + Create a child that inherits a saved isolated workspace or operator branch. Verify it continues the same files after the prior writer saves, including on a new allocation. Two bound children must not acquire concurrent writer authority. diff --git a/server/src/services/remote-workspace-snapshot-archive.test.ts b/server/src/services/remote-workspace-snapshot-archive.test.ts index 07fa20609f1..ef9664d05df 100644 --- a/server/src/services/remote-workspace-snapshot-archive.test.ts +++ b/server/src/services/remote-workspace-snapshot-archive.test.ts @@ -7,10 +7,21 @@ import { promisify } from "node:util"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { captureRemoteWorkspaceDirectory, remoteWorkspaceArchiveScript, restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js"; -const archiveIo = vi.hoisted(() => ({ afterChmod: undefined as ((target: string, mode: number) => void) | undefined })); +const archiveIo = vi.hoisted(() => ({ + afterChmod: undefined as ((target: string, mode: number) => void) | undefined, + beforeOpen: undefined as ((target: string) => Promise) | undefined, + captures: [] as string[], +})); vi.mock("node:fs/promises", async importOriginal => { const actual = await importOriginal(); - return { ...actual, async chmod(target: string, mode: number) { + return { ...actual, async open(...args: Parameters) { + await archiveIo.beforeOpen?.(String(args[0])); + return actual.open(...args); + }, async mkdtemp(...args: Parameters) { + const directory = await actual.mkdtemp(...args); + if (String(args[0]).includes("pc-workspace-capture-")) archiveIo.captures.push(String(directory)); + return directory; + }, async chmod(target: string, mode: number) { await actual.chmod(target, mode); archiveIo.afterChmod?.(target, mode); } }; }); @@ -22,7 +33,10 @@ const git = async (cwd: string, ...args: string[]) => (await execute("git", ["-C describe("portable remote workspace archives", () => { let root: string; beforeEach(async () => { root = await fs.mkdtemp(path.join(os.tmpdir(), "pc-workspace-archive-test-")); }); - afterEach(async () => { archiveIo.afterChmod = undefined; await fs.rm(root, { recursive: true, force: true }); }); + afterEach(async () => { + archiveIo.afterChmod = undefined; archiveIo.beforeOpen = undefined; archiveIo.captures = []; + await fs.rm(root, { recursive: true, force: true }); + }); async function source() { const cwd = path.join(root, "source"); await fs.mkdir(cwd); return cwd; } async function capture(cwd: string) { const archivePath = path.join(root, "snapshot.pcws"); return { archivePath, ...await captureRemoteWorkspaceDirectory({ cwd, archivePath }) }; } async function restore(snapshot: Awaited>) { @@ -57,6 +71,76 @@ describe("portable remote workspace archives", () => { const snapshot = await capture(await source()); const restored = await restore(snapshot); expect(await fs.readdir(restored)).toEqual([]); }); + it("captures with room for one archive and bounded metadata without another copy of working files", async () => { + const cwd = await source(), payload = Buffer.alloc(4 * 1024 * 1024, 123); + await fs.writeFile(path.join(cwd, "work.bin"), payload); + const archivePath = path.join(root, "snapshot.pcws"); + async function bytes(directory: string): Promise { + let total = 0; + for (const name of await fs.readdir(directory)) { + const file = path.join(directory, name), stat = await fs.lstat(file); + total += stat.isDirectory() ? await bytes(file) : stat.size; + } + return total; + } + archiveIo.beforeOpen = async target => { + if (target !== archivePath) return; + let staged = 0; + for (const directory of archiveIo.captures) staged += await bytes(directory); + if (staged + payload.length > payload.length + 128 * 1024) { + throw Object.assign(new Error("No space left for the archive"), { code: "ENOSPC" }); + } + }; + const snapshot = await capture(cwd); + archiveIo.beforeOpen = undefined; + const digest = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); + expect(digest(await fs.readFile(path.join(await restore(snapshot), "work.bin")))).toBe(digest(payload)); + expect(digest(await fs.readFile(path.join(cwd, "work.bin")))).toBe(digest(payload)); + }); + it.each(["content", "symlink", "added-file"])("rejects a concurrent %s change and removes its incomplete archive", async change => { + const cwd = await source(), file = path.join(cwd, "work.txt"), archivePath = path.join(root, "snapshot.pcws"); + await fs.writeFile(file, "original\n"); + const outside = path.join(root, "outside.txt"); await fs.writeFile(outside, "must stay outside\n"); + archiveIo.beforeOpen = async target => { + if (target !== archivePath) return; + archiveIo.beforeOpen = undefined; + if (change === "content") await fs.writeFile(file, "changed\n"); + if (change === "added-file") await fs.writeFile(path.join(cwd, "new.txt"), "new file\n"); + if (change === "symlink") { await fs.unlink(file); await fs.symlink(outside, file); } + }; + await expect(capture(cwd)).rejects.toThrow(); + await expect(fs.access(archivePath)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await fs.readFile(outside, "utf8")).toBe("must stay outside\n"); + for (const directory of archiveIo.captures) await expect(fs.access(directory)).rejects.toMatchObject({ code: "ENOENT" }); + }); + it("cleans a cancelled capture without deleting a pre-existing archive", async () => { + const cwd = await source(), archivePath = path.join(root, "snapshot.pcws"); + await fs.writeFile(path.join(cwd, "file"), "retained\n"); + const controller = new AbortController(), reason = new Error("capture cancelled"); + archiveIo.beforeOpen = async target => { if (target === archivePath) controller.abort(reason); }; + await expect(captureRemoteWorkspaceDirectory({ cwd, archivePath, signal: controller.signal })).rejects.toBe(reason); + await expect(fs.access(archivePath)).rejects.toMatchObject({ code: "ENOENT" }); + archiveIo.beforeOpen = undefined; + await fs.writeFile(archivePath, "existing archive\n"); + await expect(capture(cwd)).rejects.toMatchObject({ code: "EEXIST" }); + expect(await fs.readFile(archivePath, "utf8")).toBe("existing archive\n"); + expect(await fs.readFile(path.join(cwd, "file"), "utf8")).toBe("retained\n"); + }); + it("rejects Git reference changes during archive output and keeps the new reference", async () => { + const cwd = await source(), archivePath = path.join(root, "snapshot.pcws"); + await git(cwd, "init", "-q", "-b", "main"); + await fs.writeFile(path.join(cwd, "file"), "retained\n"); + await git(cwd, "add", "."); await git(cwd, "commit", "-qm", "base"); + const head = await git(cwd, "rev-parse", "HEAD"); + archiveIo.beforeOpen = async target => { + if (target !== archivePath) return; + archiveIo.beforeOpen = undefined; + await git(cwd, "update-ref", "refs/heads/later-work", head); + }; + await expect(capture(cwd)).rejects.toThrow("remote_workspace_archive_changed"); + await expect(fs.access(archivePath)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await git(cwd, "rev-parse", "refs/heads/later-work")).toBe(head); + }); it("rejects a Files entry limit before extraction while ordinary restores keep their full contents", async () => { const cwd = await source(); await fs.writeFile(path.join(cwd, "first.txt"), "first\n"); await fs.writeFile(path.join(cwd, "second.txt"), "second\n"); const snapshot = await capture(cwd), targetDir = path.join(root, "limited"); diff --git a/server/src/services/remote-workspace-snapshot-archive.ts b/server/src/services/remote-workspace-snapshot-archive.ts index 6d4b1552053..6c910a83ed9 100644 --- a/server/src/services/remote-workspace-snapshot-archive.ts +++ b/server/src/services/remote-workspace-snapshot-archive.ts @@ -120,9 +120,28 @@ async function workspaceArchive(input: { if (state) fail("git_operation_in_progress: finish or abort the Git operation before saving the workspace"); } }; + const readStableFile = async (source: string, consume?: (bytes: Buffer) => Promise) => { + check(); + const original = await fs.lstat(source, { bigint: true }); + const handle = await fs.open(source, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK); + try { + const before = await handle.stat({ bigint: true }); + if (!before.isFile() || original.dev !== before.dev || original.ino !== before.ino || await fs.realpath(source) !== source) fail("changed"); + const digest = crypto.createHash("sha256"); let size = 0; + for await (const bytes of handle.createReadStream({ autoClose: false, signal: input.signal })) { + check(); digest.update(bytes); size += bytes.length; + await consume?.(bytes); + } + const after = await handle.stat({ bigint: true }), current = await fs.lstat(source, { bigint: true }); + if (size !== Number(before.size) || await fs.realpath(source) !== source + || ["dev", "ino", "size", "mode", "mtimeNs", "ctimeNs"].some(key => before[key as keyof typeof before] !== after[key as keyof typeof after] + || after[key as keyof typeof after] !== current[key as keyof typeof current])) fail("changed"); + return { sha256: digest.digest("hex"), size, mode: Number(before.mode & 0o777n) }; + } finally { await handle.close(); } + }; const directories: string[] = []; const excludedPaths = new Set(); - const walk = async (root: string, copyTo?: string, skipGit = false) => { + const walk = async (root: string, skipGit = false) => { const entries: Entry[] = []; const visit = async (relative: string, parentIsRepository = false) => { const directory = relative ? path.join(root, relative) : root; @@ -137,7 +156,7 @@ async function workspaceArchive(input: { if (input.exclude?.some(excluded => next === excluded || next.startsWith(`${excluded}/`)) || next === ".paperclip-runtime" || next.startsWith(".paperclip-runtime/")) { excludedPaths.add(next); continue; } - const source = path.join(root, next), target = copyTo ? path.join(copyTo, next) : null; + const source = path.join(root, next); const stat = await fs.lstat(source); if (skipGit && name === ".git") { if (stat.isSymbolicLink()) fail("unsafe_git_link"); directories.push(relative); continue; } if (stat.isDirectory() && name === "node_modules") { @@ -150,30 +169,13 @@ async function workspaceArchive(input: { } if (stat.isDirectory()) { entries.push({ path: next, type: "directory", mode: stat.mode & 0o777 }); - if (target) await fs.mkdir(target, { mode: 0o700 }); await visit(next, inRepository); } else if (stat.isSymbolicLink()) { const link = await fs.readlink(source); if (!safeLink(next, link)) fail("unsafe_symlink"); entries.push({ path: next, type: "symlink", target: link }); - if (target) await fs.symlink(link, target); } else if (stat.isFile()) { - const handle = await fs.open(source, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK); - let output: Awaited> | undefined; - try { - const before = await handle.stat({ bigint: true }); - if (!before.isFile() || BigInt(stat.dev) !== before.dev || BigInt(stat.ino) !== before.ino || await fs.realpath(source) !== source) fail("changed"); - if (target) output = await fs.open(target, "wx", 0o600); - const digest = crypto.createHash("sha256"); let bytes = 0; - for await (const part of handle.createReadStream({ autoClose: false, signal: input.signal })) { - check(); digest.update(part); bytes += part.length; - await output?.writeFile(part); - } - const after = await handle.stat({ bigint: true }), current = await fs.lstat(source, { bigint: true }); - if (bytes !== Number(before.size) || await fs.realpath(source) !== source || ["dev", "ino", "size", "mtimeNs", "ctimeNs"].some(key => before[key as keyof typeof before] !== after[key as keyof typeof after] || after[key as keyof typeof after] !== current[key as keyof typeof current])) fail("changed"); - entries.push({ path: next, type: "file", mode: stat.mode & 0o777, size: bytes, sha256: digest.digest("hex") }); - if (target) await fs.chmod(target, stat.mode & 0o777); - } finally { await output?.close(); await handle.close(); } + entries.push({ path: next, type: "file", ...await readStableFile(source) }); } else fail("unsupported_entry"); } }; @@ -188,9 +190,10 @@ async function workspaceArchive(input: { const root = await fs.realpath(input.cwd!); if (!(await fs.lstat(root)).isDirectory()) fail("source_invalid"); const scratch = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "pc-workspace-capture-"))); + let archiveCreated = false, completed = false; try { const tree = path.join(scratch, "tree"); await fs.mkdir(tree, { mode: 0o700 }); - const before = await walk(root, tree, true); + const before = await walk(root, true); const repositories = [...new Set(directories)]; const gitSignatures = new Map(); const signature = async (repo: string, settings?: Awaited>) => { @@ -207,6 +210,7 @@ async function workspaceArchive(input: { // private repository. The selected worktree's index and HEAD are separate. await git(root, ["clone", "--mirror", "--no-hardlinks", "--dissociate", "--", repo, normalized]); const destination = path.join(tree, relative, ".git"); + await fs.mkdir(path.dirname(destination), { recursive: true, mode: 0o700 }); await fs.rename(normalized, destination); const sourceGit = (await git(repo, ["rev-parse", "--absolute-git-dir"])).trim(); const sourceCommon = (await git(repo, ["rev-parse", "--path-format=absolute", "--git-common-dir"])).trim(); @@ -232,35 +236,38 @@ async function workspaceArchive(input: { const [object, ref] = row.split(" "); return `update ${ref} ${object}\n`; }).join(""); if (updates) await git(restoredRepo, ["update-ref", "--stdin"], updates); - await git(restoredRepo, ["repack", "-a", "-d"]); await fs.rm(path.join(destination, "objects", "info", "alternates"), { force: true }); await git(restoredRepo, ["fsck", "--connectivity-only", "--no-dangling"]); if (await signature(repo) !== gitSignatures.get(relative)) fail("changed"); } - directories.length = 0; - const after = await walk(root, undefined, true); - if (JSON.stringify(before) !== JSON.stringify(after)) fail("changed"); - for (const relative of repositories) await assertGitOperationComplete(path.join(root, relative)); - const entries = await walk(tree); - // Keep private staging writable for cleanup; the manifest preserves the - // source modes and restoration applies them only after writing children. const originals = new Map(before.map(entry => [entry.path, entry])); - for (const entry of entries) if (entry.type === "directory") { - const original = originals.get(entry.path); if (original) entry.mode = original.mode; - } + const entries = [...before, ...(await walk(tree)).filter(entry => !originals.has(entry.path))] + .sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0); const manifest = { schema, exclusionPolicy: "paperclip.remote-workspace-exclusions.v1", excluded: [...excludedPaths].sort(), entries }, body = Buffer.from(JSON.stringify(manifest)); if (body.length > 64 * 1024 * 1024) fail("manifest_too_large"); const digest = manifestHash(manifest), length = Buffer.alloc(4); length.writeUInt32BE(body.length); const output = await fs.open(input.archivePath, "wx", 0o600); + archiveCreated = true; try { await output.writeFile(Buffer.concat([magic, length, body])); for (const entry of entries) if (entry.type === "file") { - for await (const bytes of createReadStream(path.join(tree, entry.path), { signal: input.signal })) { check(); await output.writeFile(bytes); } + const observed = await readStableFile(path.join(originals.has(entry.path) ? root : tree, entry.path), bytes => output.writeFile(bytes)); + if (observed.size !== entry.size || observed.sha256 !== entry.sha256 || observed.mode !== entry.mode) fail("changed"); } } finally { await output.close(); } + directories.length = 0; + if (JSON.stringify(before) !== JSON.stringify(await walk(root, true))) fail("changed"); + for (const relative of repositories) { + if (await signature(path.join(root, relative)) !== gitSignatures.get(relative)) fail("changed"); + } if (input.manifestPath) await fs.writeFile(input.manifestPath, body, { flag: "wx", mode: 0o600 }); - return { schema, manifestDigest: digest, ...await hashFile(input.archivePath), entries: entries.length }; - } finally { await fs.rm(scratch, { recursive: true, force: true }); } + const content = await hashFile(input.archivePath); + completed = true; + return { schema, manifestDigest: digest, ...content, entries: entries.length }; + } finally { + if (archiveCreated && !completed) await fs.rm(input.archivePath, { force: true }); + await fs.rm(scratch, { recursive: true, force: true }); + } } if (!input.targetDir || !/^[a-f0-9]{64}$/.test(input.expectedManifestDigest ?? "")) fail("restore_binding_invalid"); if (input.maxEntries !== undefined && (!Number.isSafeInteger(input.maxEntries) || input.maxEntries < 0)) fail("restore_binding_invalid");