fix(runtime): admit manifests against available disk capacity

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/DEVELOPING.md

@@ -707,7 +707,7 @@ Paperclip applies one process-wide scheduler to expensive host-side workspace Gi
 Workspace snapshots list ignored paths with `git ls-files --others --ignored --exclude-standard --directory -z` so ignored directory contents do not require a full status walk. Changed, untracked, deleted, and ignored filename lists stream into private SQLite manifests. There is no total filename-list byte limit. The parser and each sink chunk are limited to 64 KiB, and each SQLite connection uses a 1 MiB page cache with disk-backed temporary storage. Records must be complete NUL-delimited UTF-8 paths. Invalid, oversized, or incomplete records fail the scan. Explicit file selection excludes files created after the scan. Buffered browser/guard and referenced-source scan bounds stay unchanged. Snapshot failures retain their typed cause instead of becoming a non-Git-folder result. During pre-provider setup, scan timeouts and queue saturation use the existing two automatic failure retries with a 30-second delay. Cancellation, output limits, and other Git errors stop with specific recovery guidance. See `doc/execution-semantics.md` for the ownership and retry-budget contract.
 
 
-Snapshot scans default to a 30-minute execution deadline, including disk backpressure, rather than the short interactive scan deadline. Set `PAPERCLIP_WORKSPACE_GIT_SNAPSHOT_TIMEOUT_MS` to allow slower storage or larger trees (1 second to 24 hours). Disk capacity, filesystem path limits, Git's own resource use, and this execution budget remain practical limits. Manifest writers check available disk space before creation and at most every 1 MiB of input thereafter. They stop below a 256 MiB free-space reserve. `PAPERCLIP_WORKSPACE_MANIFEST_MIN_FREE_BYTES` can raise or lower the reserve, with a 64 MiB minimum. Concurrent writers can consume space between checks; filesystem quota and write errors still fail the operation. The scheduler keeps the active slot until the child, pipes, and pending sink write finish. Snapshot failures cancel sibling producers and settle all started producers before removing temporary storage.
+Snapshot scans default to a 30-minute execution deadline, including disk backpressure, rather than the short interactive scan deadline. Set `PAPERCLIP_WORKSPACE_GIT_SNAPSHOT_TIMEOUT_MS` to allow slower storage or larger trees (1 second to 24 hours). Disk capacity, filesystem path limits, Git's own resource use, and this execution budget remain practical limits. Each manifest is admitted with a page allowance equal to one quarter of the available bytes above the host reserve at creation. SQLite enforces that allowance before page allocation, so one snapshot cannot consume all available storage. This is a disk-capacity admission policy, not a fixed filename-list byte or record limit. Nested repository and baseline manifests receive separate allowances from the remaining capacity. Manifest writers also check available disk space before creation and at most every 1 MiB of input thereafter. They stop below a 256 MiB free-space reserve. `PAPERCLIP_WORKSPACE_MANIFEST_MIN_FREE_BYTES` can raise or lower the reserve, with a 64 MiB minimum. Concurrent writers can consume space between checks; filesystem quota and write errors still fail the operation. The scheduler keeps the active slot until the child, pipes, and pending sink write finish. Snapshot failures cancel sibling producers and settle all started producers before removing temporary storage.
 
 Workspace baselines, nested-repository subsets, and merge lookups also use disk-backed manifests. Local archive input uses NUL list files. Deletion replay uploads a NUL manifest and runs bounded argument batches with ancestor symlink checks. Git ignored paths are literal indexed exclusions for staging, baseline capture, and the host merge. Restore transport uses the fixed/configured archive exclusions; it can transfer a remotely created ignored file, but the host merge excludes it. The v2 native recovery descriptor contains manifest references, not file arrays. Recovery checks controller-owned paths and content digests before reading records. Existing v1 descriptors remain readable. Transient manifests are removed on preparation failure and restore completion; native manifests remain until the durable recovery lifecycle is cleaned up.
 

packages/adapter-utils/src/workspace-manifest.test.ts

@@ -10,6 +10,7 @@ const cleanup: string[] = [];
 afterEach(async () => {
   setExpensiveWorkspaceGitExecutor(null);
   vi.restoreAllMocks();
+  vi.unstubAllEnvs();
   for (const dir of cleanup.splice(0)) await fs.rm(dir, { recursive: true, force: true });
 });
 
@@ -52,6 +53,48 @@ it("refuses new manifests and fails ongoing writes when the disk reserve is exha
   } finally { writer.close(false); }
 });
 
+it("admits each snapshot against current disk capacity and rejects growth at its page allowance", async () => {
+  const available = nodeFs.statfsSync(os.tmpdir(), { bigint: true });
+  const reserve = 256 * 1024 * 1024;
+  vi.stubEnv("PAPERCLIP_WORKSPACE_MANIFEST_MIN_FREE_BYTES", String(reserve));
+  const capacity = reserve + 128 * 1024;
+  vi.spyOn(nodeFs, "statfsSync").mockReturnValue({ ...available, bsize: 1n, bavail: BigInt(capacity) } as never);
+  for (let attempt = 0; attempt < 3; attempt++) {
+    const writer = await createWorkspaceManifest();
+    cleanup.push(path.dirname(writer.filePath));
+    try {
+      expect(() => writer.batch(() => {
+        for (let index = 0; index < 100; index++) writer.add("overlay", `${index}-${"x".repeat(2048)}`);
+      })).toThrow(/full/i);
+      expect((await fs.stat(writer.filePath)).size).toBeLessThanOrEqual(32 * 1024);
+    } finally { writer.close(false); }
+  }
+});
+
+it("fails a real Git scan at its disk allowance and removes the incomplete manifest", async () => {
+  const repo = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-scan-quota-"));
+  cleanup.push(repo);
+  await runLocalGit(repo, ["init"]);
+  await runLocalGit(repo, ["commit", "--allow-empty", "-qm", "fixture"], {
+    env: { ...process.env, GIT_AUTHOR_NAME: "Test", GIT_COMMITTER_NAME: "Test", GIT_AUTHOR_EMAIL: "test@example.test", GIT_COMMITTER_EMAIL: "test@example.test" },
+  });
+  for (let index = 0; index < 500; index++) await fs.writeFile(path.join(repo, `${index}-${"x".repeat(200)}`), "");
+  const available = nodeFs.statfsSync(os.tmpdir(), { bigint: true });
+  const reserve = 256 * 1024 * 1024;
+  vi.stubEnv("PAPERCLIP_WORKSPACE_MANIFEST_MIN_FREE_BYTES", String(reserve));
+  vi.spyOn(nodeFs, "statfsSync").mockReturnValue({ ...available, bsize: 1n, bavail: BigInt(reserve + 128 * 1024) } as never);
+  const temporary: string[] = [];
+  const mkdtemp = fs.mkdtemp.bind(fs);
+  vi.spyOn(fs, "mkdtemp").mockImplementation(async (prefix, options) => {
+    const directory = await mkdtemp(prefix, options);
+    temporary.push(String(directory));
+    return directory;
+  });
+  await expect(readGitWorkspaceSnapshot(repo)).rejects.toMatchObject({ code: "workspace_git_scan_failed" });
+  expect(temporary).toHaveLength(1);
+  await expect(fs.stat(temporary[0]!)).rejects.toMatchObject({ code: "ENOENT" });
+});
+
 it("settles a late producer before removing shared storage after another scan fails", async () => {
   const repo = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-scan-barrier-"));
   cleanup.push(repo);

packages/adapter-utils/src/workspace-manifest.ts

@@ -11,7 +11,7 @@ export const WORKSPACE_STREAM_CHUNK_BYTES = 64 * 1024;
 const DISK_CHECK_INTERVAL_BYTES = 1024 * 1024;
 
 /** A capacity floor, not a total filename budget. Leave room for the host. */
-export function assertWorkspaceManifestDiskSpace(directory: string, requiredBytes = 0): void {
+export function assertWorkspaceManifestDiskSpace(directory: string, requiredBytes = 0): number {
   const configured = Number(process.env.PAPERCLIP_WORKSPACE_MANIFEST_MIN_FREE_BYTES);
   const minimum = Number.isSafeInteger(configured) && configured >= 64 * 1024 * 1024
     ? configured : 256 * 1024 * 1024;
@@ -21,6 +21,7 @@ export function assertWorkspaceManifestDiskSpace(directory: string, requiredByte
       code: "workspace_git_scan_failed",
     });
   }
+  return Number(stat.bavail * stat.bsize - BigInt(minimum));
 }
 export interface PathManifest {
   kind: "path_manifest";
@@ -58,9 +59,15 @@ export class WorkspaceManifestWriter {
   private closed = false;
   private bytesSinceDiskCheck = 0;
   constructor(readonly filePath: string) {
-    assertWorkspaceManifestDiskSpace(path.dirname(filePath));
+    const availableBytes = assertWorkspaceManifestDiskSpace(path.dirname(filePath));
     this.db = openDatabase(filePath, false);
     try {
+      // Admission follows current storage capacity, not filename-list length.
+      // SQLite enforces this before allocating pages, including within a chunk.
+      const pageSize = Number(this.db.prepare("PRAGMA page_size").get()!.page_size);
+      const pageAllowance = Math.floor(availableBytes / 4 / pageSize);
+      if (pageAllowance < 2) throw new Error("Insufficient workspace manifest disk allowance");
+      this.db.exec(`PRAGMA max_page_count=${Math.min(pageAllowance, 4_294_967_294)}`);
       this.db.exec("PRAGMA journal_mode=DELETE; CREATE TABLE records(category TEXT, path TEXT, value TEXT NOT NULL, PRIMARY KEY(category,path)) WITHOUT ROWID;");
       this.insert = this.db.prepare("INSERT OR REPLACE INTO records VALUES(?,?,?)");
       this.db.exec("BEGIN");
@@ -79,7 +86,11 @@ export class WorkspaceManifestWriter {
   batch(work: () => void): void {
     this.db.exec("SAVEPOINT chunk");
     try { work(); this.db.exec("RELEASE chunk"); }
-    catch (error) { this.db.exec("ROLLBACK TO chunk; RELEASE chunk"); throw error; }
+    catch (error) {
+      // SQLITE_FULL can roll back the transaction itself.
+      if (this.db.isTransaction) this.db.exec("ROLLBACK TO chunk; RELEASE chunk");
+      throw error;
+    }
   }
   paths(category: string): PathManifest {
     const row = this.db.prepare("SELECT count(*) AS count FROM records WHERE category=?").get(category)!;
@@ -88,7 +99,7 @@ export class WorkspaceManifestWriter {
   close(commit = true): void {
     if (this.closed) return;
     this.closed = true;
-    try { this.db.exec(commit ? "COMMIT" : "ROLLBACK"); }
+    try { if (this.db.isTransaction) this.db.exec(commit ? "COMMIT" : "ROLLBACK"); }
     finally { this.db.close(); }
   }
 }