diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 8c5bc2b0d71..ebd7fb133db 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -707,7 +707,7 @@ Paperclip applies one process-wide scheduler to expensive host-side workspace Gi Workspace snapshots list ignored paths with `git ls-files --others --ignored --exclude-standard --directory -z` so ignored directory contents do not require a full status walk. Changed, untracked, deleted, and ignored filename lists stream into private SQLite manifests. There is no total filename-list byte limit. The parser and each sink chunk are limited to 64 KiB, and each SQLite connection uses a 1 MiB page cache with disk-backed temporary storage. Records must be complete NUL-delimited UTF-8 paths. Invalid, oversized, or incomplete records fail the scan. Explicit file selection excludes files created after the scan. Buffered browser/guard and referenced-source scan bounds stay unchanged. Snapshot failures retain their typed cause instead of becoming a non-Git-folder result. During pre-provider setup, scan timeouts and queue saturation use the existing two automatic failure retries with a 30-second delay. Cancellation, output limits, and other Git errors stop with specific recovery guidance. See `doc/execution-semantics.md` for the ownership and retry-budget contract. -Snapshot scans default to a 30-minute execution deadline, including disk backpressure, rather than the short interactive scan deadline. Set `PAPERCLIP_WORKSPACE_GIT_SNAPSHOT_TIMEOUT_MS` to allow slower storage or larger trees (1 second to 24 hours). Disk capacity, filesystem path limits, Git's own resource use, and this execution budget remain practical limits. The scheduler keeps the active slot until the child, pipes, and pending sink write finish. Snapshot failures cancel sibling producers and settle all started producers before removing temporary storage. +Snapshot scans default to a 30-minute execution deadline, including disk backpressure, rather than the short interactive scan deadline. Set `PAPERCLIP_WORKSPACE_GIT_SNAPSHOT_TIMEOUT_MS` to allow slower storage or larger trees (1 second to 24 hours). Disk capacity, filesystem path limits, Git's own resource use, and this execution budget remain practical limits. Manifest writers check available disk space before creation and at most every 1 MiB of input thereafter. They stop below a 256 MiB free-space reserve. `PAPERCLIP_WORKSPACE_MANIFEST_MIN_FREE_BYTES` can raise or lower the reserve, with a 64 MiB minimum. Concurrent writers can consume space between checks; filesystem quota and write errors still fail the operation. The scheduler keeps the active slot until the child, pipes, and pending sink write finish. Snapshot failures cancel sibling producers and settle all started producers before removing temporary storage. Workspace baselines, nested-repository subsets, and merge lookups also use disk-backed manifests. Local archive input uses NUL list files. Deletion replay uploads a NUL manifest and runs bounded argument batches with ancestor symlink checks. Git ignored paths are literal indexed exclusions for staging, baseline capture, and the host merge. Restore transport uses the fixed/configured archive exclusions; it can transfer a remotely created ignored file, but the host merge excludes it. The v2 native recovery descriptor contains manifest references, not file arrays. Recovery checks controller-owned paths and content digests before reading records. Existing v1 descriptors remain readable. Transient manifests are removed on preparation failure and restore completion; native manifests remain until the durable recovery lifecycle is cleaned up. diff --git a/packages/adapter-utils/src/workspace-manifest.test.ts b/packages/adapter-utils/src/workspace-manifest.test.ts index f08dc049234..8fadc360d5a 100644 --- a/packages/adapter-utils/src/workspace-manifest.test.ts +++ b/packages/adapter-utils/src/workspace-manifest.test.ts @@ -1,4 +1,5 @@ import fs from "node:fs/promises"; +import nodeFs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, expect, it, vi } from "vitest"; @@ -36,6 +37,21 @@ it("deduplicates paths on disk and rejects incomplete manifest metadata", async expect(() => [...workspacePaths(["../escape"])]).toThrow("Invalid"); }); +it("refuses new manifests and fails ongoing writes when the disk reserve is exhausted", async () => { + const writer = await createWorkspaceManifest(); + cleanup.push(path.dirname(writer.filePath)); + const available = nodeFs.statfsSync(os.tmpdir(), { bigint: true }); + vi.spyOn(nodeFs, "statfsSync").mockReturnValue({ ...available, bavail: 0n } as never); + try { + for (let attempt = 0; attempt < 4; attempt++) { + await expect(createWorkspaceManifest()).rejects.toMatchObject({ code: "workspace_git_scan_failed" }); + } + expect(() => { + for (let index = 0; index < 1024; index++) writer.add("overlay", `${index}-${"x".repeat(2048)}`); + }).toThrow("free-space reserve"); + } finally { writer.close(false); } +}); + it("settles a late producer before removing shared storage after another scan fails", async () => { const repo = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-scan-barrier-")); cleanup.push(repo); diff --git a/packages/adapter-utils/src/workspace-manifest.ts b/packages/adapter-utils/src/workspace-manifest.ts index ae73bd387c0..7ab40bb84e4 100644 --- a/packages/adapter-utils/src/workspace-manifest.ts +++ b/packages/adapter-utils/src/workspace-manifest.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; import { constants, promises as fs } from "node:fs"; +import nodeFs from "node:fs"; import os from "node:os"; import path from "node:path"; import { DatabaseSync, type StatementSync } from "node:sqlite"; @@ -7,6 +8,20 @@ import { DatabaseSync, type StatementSync } from "node:sqlite"; // Limits apply to one record and to SQLite's page cache, never to the list. export const WORKSPACE_PATH_MAX_BYTES = 64 * 1024; export const WORKSPACE_STREAM_CHUNK_BYTES = 64 * 1024; +const DISK_CHECK_INTERVAL_BYTES = 1024 * 1024; + +/** A capacity floor, not a total filename budget. Leave room for the host. */ +export function assertWorkspaceManifestDiskSpace(directory: string, requiredBytes = 0): void { + const configured = Number(process.env.PAPERCLIP_WORKSPACE_MANIFEST_MIN_FREE_BYTES); + const minimum = Number.isSafeInteger(configured) && configured >= 64 * 1024 * 1024 + ? configured : 256 * 1024 * 1024; + const stat = nodeFs.statfsSync(directory, { bigint: true }); + if (stat.bavail * stat.bsize < BigInt(minimum) + BigInt(requiredBytes)) { + throw Object.assign(new Error("Workspace manifest storage is below its free-space reserve"), { + code: "workspace_git_scan_failed", + }); + } +} export interface PathManifest { kind: "path_manifest"; version: 1; @@ -41,7 +56,9 @@ export class WorkspaceManifestWriter { private readonly db: DatabaseSync; private readonly insert: StatementSync; private closed = false; + private bytesSinceDiskCheck = 0; constructor(readonly filePath: string) { + assertWorkspaceManifestDiskSpace(path.dirname(filePath)); this.db = openDatabase(filePath, false); try { this.db.exec("PRAGMA journal_mode=DELETE; CREATE TABLE records(category TEXT, path TEXT, value TEXT NOT NULL, PRIMARY KEY(category,path)) WITHOUT ROWID;"); @@ -52,6 +69,11 @@ export class WorkspaceManifestWriter { add(category: string, relative: string, value = ""): void { assertWorkspaceRelativePath(relative); if (Buffer.byteLength(value) > WORKSPACE_PATH_MAX_BYTES) throw new Error("Workspace manifest value exceeds record limit"); + this.bytesSinceDiskCheck += Buffer.byteLength(relative) + Buffer.byteLength(value); + if (this.bytesSinceDiskCheck >= DISK_CHECK_INTERVAL_BYTES) { + assertWorkspaceManifestDiskSpace(path.dirname(this.filePath)); + this.bytesSinceDiskCheck = 0; + } this.insert.run(category, relative, value); } batch(work: () => void): void { diff --git a/packages/adapter-utils/src/workspace-restore-merge.test.ts b/packages/adapter-utils/src/workspace-restore-merge.test.ts index e82054e5fd7..7833997dd44 100644 --- a/packages/adapter-utils/src/workspace-restore-merge.test.ts +++ b/packages/adapter-utils/src/workspace-restore-merge.test.ts @@ -10,6 +10,7 @@ import { resolvePaperclipInstanceRootForAdapter } from "./server-utils.js"; import { captureDirectorySnapshot, directorySnapshotSha256, + disposeDirectorySnapshot, classifyWorkspaceRestoreFailure, describeWorkspaceRestoreFailure, mergeDirectoryWithBaseline, @@ -104,6 +105,24 @@ describe("workspace restore merge", () => { ).resolves.toBe("ssh codex\n"); }); + it("preserves a host file replacing a deleted baseline directory and continues the restore", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-restore-conflict-")); + cleanupDirs.push(rootDir); + const targetDir = path.join(rootDir, "target"); + const sourceDir = path.join(rootDir, "source"); + await mkdir(path.join(targetDir, "replaced", "nested"), { recursive: true }); + await mkdir(sourceDir); + const baseline = await captureDirectorySnapshot(targetDir, { exclude: [], diskBacked: true }); + try { + await rm(path.join(targetDir, "replaced"), { recursive: true }); + await writeFile(path.join(targetDir, "replaced"), "host change"); + await writeFile(path.join(sourceDir, "other.txt"), "sandbox change"); + await mergeDirectoryWithBaseline({ baseline, sourceDir, targetDir }); + expect(await readFile(path.join(targetDir, "replaced"), "utf8")).toBe("host change"); + expect(await readFile(path.join(targetDir, "other.txt"), "utf8")).toBe("sandbox change"); + } finally { await disposeDirectorySnapshot(baseline); } + }); + it("ignores non-file entries when capturing snapshots", async () => { if (process.platform === "win32") return; diff --git a/packages/adapter-utils/src/workspace-restore-merge.ts b/packages/adapter-utils/src/workspace-restore-merge.ts index 073d2833f32..88e89e8f920 100644 --- a/packages/adapter-utils/src/workspace-restore-merge.ts +++ b/packages/adapter-utils/src/workspace-restore-merge.ts @@ -533,7 +533,7 @@ export async function mergeDirectoryWithBaseline(input: { // Reverse path order visits descendants before their parent directory. for (const [relative, entry] of orderedEntries(input.baseline, true)) { if (entry.kind === "dir" && !source.entries.has(relative)) await fs.rmdir(path.join(canonicalTargetDir, relative)).catch((error: NodeJS.ErrnoException) => { - if (error.code !== "ENOENT" && error.code !== "ENOTEMPTY") throw error; + if (error.code !== "ENOENT" && error.code !== "ENOTEMPTY" && error.code !== "ENOTDIR") throw error; }); } for (const [relative, entry] of orderedEntries(source)) { diff --git a/server/src/services/native-runtime/native-workspace-sync.ts b/server/src/services/native-runtime/native-workspace-sync.ts index 8bf13711c37..a618f612ebc 100644 --- a/server/src/services/native-runtime/native-workspace-sync.ts +++ b/server/src/services/native-runtime/native-workspace-sync.ts @@ -25,7 +25,7 @@ import type { WorkspaceDurableSeedPaths, WorkspaceInboundMode, } from "@paperclipai/adapter-utils/sandbox-managed-runtime"; -import { isPathManifest, manifestFileSha256, readManifestRecords, type PathManifest, type WorkspacePaths } from "@paperclipai/adapter-utils/workspace-manifest"; +import { assertWorkspaceManifestDiskSpace, isPathManifest, manifestFileSha256, readManifestRecords, type PathManifest, type WorkspacePaths } from "@paperclipai/adapter-utils/workspace-manifest"; import { resolvePaperclipInstanceRoot } from "../../home-paths.js"; import { parseObject } from "../../adapters/utils.js"; import type { NativeRestartRecoveryClaim } from "./native-restart-recovery.js"; @@ -370,6 +370,9 @@ async function persistSnapshotManifests(runId: string, snapshot: NonNullable
 {
+  try { await exec(command, args, options); }
+  catch (error) {
+    const stderr = String((error as { stderr?: unknown }).stderr ?? "");
+    throw new Error(`Fixture command failed: ${String(error)}\n${stderr.slice(-8192)}`, { cause: error });
+  }
+}
 const directories: string[] = [];
 const snapshots: GitWorkspaceSnapshot[] = [];
 const runtimes: PreparedSandboxManagedRuntime[] = [];
@@ -25,6 +32,7 @@ function localClient(commands: string[]): SandboxManagedRuntimeClient {
     makeDir: async (dir) => { await fs.mkdir(dir, { recursive: true }); },
     writeFile: async (file, bytes) => { await fs.writeFile(file, Buffer.from(bytes)); },
     readFile: async (file) => fs.readFile(file),
+    listFiles: async (dir) => fs.readdir(dir),
     remove: async (file) => { await fs.rm(file, { recursive: true, force: true }); },
     run: async (command) => { await execute("sh", ["-c", command]); },
     syncIn: async (operations) => {