fix(runtime): reserve worst-case PAX archive overhead

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md

@@ -183,6 +183,12 @@ exclusive maximum-size reservation so those files are preserved. Enforce the
 reservation while streaming the archive. Ordinary runtime backups keep their
 existing maximum archive limit.
 
+Derive tar overhead from the maximum entry count and both maximum path lengths.
+PAX can encode a path and symlink target for every entry. The difference between
+the global object and content limits is not a safe per-source overhead estimate.
+Check the bound against real maximum-path PAX encoding at the full entry budget,
+and check streaming at an exact byte limit and one byte beyond it.
+
 On failure, cancel and await every transfer in the group before returning. A
 completed publication remains valid for the next attempt. Tests must exercise
 real PostgreSQL publication and byte restoration, the four-transfer bound,

server/src/services/native-runtime/native-harness-backup-archive.test.ts

@@ -5,7 +5,9 @@ import { tmpdir } from "node:os";
 import { pipeline } from "node:stream/promises";
 import tar from "tar-stream";
 import { afterEach, expect, it } from "vitest";
-import { captureNativeHarnessArchive, restoreNativeHarnessArchive, MAX_NATIVE_HARNESS_BYTES } from "./native-harness-backup-archive.js";
+import { captureNativeHarnessArchive, restoreNativeHarnessArchive, MAX_NATIVE_HARNESS_BYTES,
+  MAX_NATIVE_HARNESS_ENTRIES, MAX_NATIVE_HARNESS_OBJECT_BYTES, MAX_NATIVE_HARNESS_PATH_BYTES,
+  nativeHarnessArchiveSizeLimit } from "./native-harness-backup-archive.js";
 
 const roots: string[] = [];
 afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); });
@@ -16,6 +18,32 @@ async function maliciousArchive(root: string, headers: Array<tar.Headers>) {
   for (const header of headers) archive.entry(header, Buffer.alloc(0));
   archive.finalize(); await written; return file;
 }
+it("reserves the full entry budget using real maximum-path PAX encoding", async () => {
+  const root = await fixture(), name = "a/".repeat(MAX_NATIVE_HARNESS_PATH_BYTES / 2 - 1) + "aa";
+  const file = await maliciousArchive(root, [{ name, type: "symlink", linkname: "b".repeat(MAX_NATIVE_HARNESS_PATH_BYTES) }]);
+  // One real encoded record gives the repeated-entry size without creating a
+  // gigabyte fixture. Each hypothetical entry includes its own PAX records.
+  const entryBytes = (await stat(file)).size - 1024;
+  const denseArchiveBytes = entryBytes * MAX_NATIVE_HARNESS_ENTRIES + 1024;
+  const bound = nativeHarnessArchiveSizeLimit(0);
+  expect(denseArchiveBytes).toBeGreaterThan(MAX_NATIVE_HARNESS_OBJECT_BYTES - MAX_NATIVE_HARNESS_BYTES);
+  expect(denseArchiveBytes).toBeLessThanOrEqual(bound);
+  expect(denseArchiveBytes).toBeGreaterThan(bound * 0.9);
+  expect(nativeHarnessArchiveSizeLimit(MAX_NATIVE_HARNESS_BYTES)).toBe(MAX_NATIVE_HARNESS_OBJECT_BYTES);
+});
+it("captures at the exact archive byte limit and refuses one byte beyond it", async () => {
+  const root = await fixture(), source = join(root, "source"); await mkdir(source);
+  await writeFile(join(source, "manifest.json"), "{}"); await writeFile(join(source, "state"), "retained provider bytes");
+  const original = join(root, "original"); await captureNativeHarnessArchive(source, original);
+  const bytes = (await stat(original)).size, exact = join(root, "exact");
+  await captureNativeHarnessArchive(source, exact, undefined, bytes);
+  expect((await stat(exact)).size).toBe(bytes);
+  await restoreNativeHarnessArchive(exact, join(root, "restored-exact"));
+  expect(await readFile(join(root, "restored-exact", "state"), "utf8")).toBe("retained provider bytes");
+  const exceeded = join(root, "exceeded");
+  await expect(captureNativeHarnessArchive(source, exceeded, undefined, bytes - 1)).rejects.toThrow("native_harness_archive_invalid");
+  expect((await stat(exceeded)).size).toBeLessThan(bytes);
+});
 it.each([
   { name: "../escaped", type: "file" as const },
   { name: "/absolute", type: "file" as const },

server/src/services/native-runtime/native-harness-backup-archive.ts

@@ -8,12 +8,22 @@ import tar from "tar-stream";
 export const MAX_NATIVE_HARNESS_BYTES = 4 * 1024 * 1024 * 1024;
 export const MAX_NATIVE_HARNESS_OBJECT_BYTES = MAX_NATIVE_HARNESS_BYTES + 768 * 1024 * 1024;
 export const MAX_NATIVE_HARNESS_ENTRIES = 100_000;
+export const MAX_NATIVE_HARNESS_PATH_BYTES = 4096;
+// PAX can store both a full path and a full symlink target for each entry.
+// Two KiB also cover PAX framing, both headers and payload padding. The final
+// two zero blocks terminate the archive. Keep this separate from content size.
+export const MAX_NATIVE_HARNESS_TAR_OVERHEAD_BYTES = MAX_NATIVE_HARNESS_ENTRIES
+  * (2 * MAX_NATIVE_HARNESS_PATH_BYTES + 2048) + 1024;
 const fail = () => new Error("native_harness_archive_invalid");
-const safeName = (name: string) => name.length > 0 && Buffer.byteLength(name) <= 4096 && !name.includes("\\")
+export function nativeHarnessArchiveSizeLimit(contentBytes: number) {
+  if (!Number.isSafeInteger(contentBytes) || contentBytes < 0 || contentBytes > MAX_NATIVE_HARNESS_BYTES) throw fail();
+  return Math.min(MAX_NATIVE_HARNESS_OBJECT_BYTES, contentBytes + MAX_NATIVE_HARNESS_TAR_OVERHEAD_BYTES);
+}
+const safeName = (name: string) => name.length > 0 && Buffer.byteLength(name) <= MAX_NATIVE_HARNESS_PATH_BYTES && !name.includes("\\")
   && !/[\u0000-\u001f\u007f]/u.test(name) && !posix.isAbsolute(name)
   && name.split("/").every(part => part !== ".." && part !== "." && part !== "");
 
-const safeLink = (name: string, target: string) => target.length > 0 && Buffer.byteLength(target) <= 4096
+const safeLink = (name: string, target: string) => target.length > 0 && Buffer.byteLength(target) <= MAX_NATIVE_HARNESS_PATH_BYTES
   && !target.includes("\\") && !/[\u0000-\u001f\u007f]/u.test(target) && !posix.isAbsolute(target)
   && !["..", "."].includes(posix.normalize(posix.join(posix.dirname(name), target)))
   && !posix.normalize(posix.join(posix.dirname(name), target)).startsWith("../");

server/src/services/native-runtime/native-harness-backup-migration.ts

@@ -6,7 +6,7 @@ import { environmentLeases, heartbeatRuns, nativeHarnessBackups, type Db } from
 import type { StorageService } from "../../storage/types.js";
 import { nativeHarnessStateBase, readNativeHarnessBackupStamp, verifyNativeHarnessBackupStampAsync } from "./native-harness-backup-stamp.js";
 import { persistNativeHarnessBackup, verifyDurableNativeHarnessBackup } from "./native-harness-backup-storage.js";
-import { MAX_NATIVE_HARNESS_BYTES, MAX_NATIVE_HARNESS_OBJECT_BYTES } from "./native-harness-backup-archive.js";
+import { MAX_NATIVE_HARNESS_BYTES, MAX_NATIVE_HARNESS_OBJECT_BYTES, nativeHarnessArchiveSizeLimit } from "./native-harness-backup-archive.js";
 import { offlineMigrationExecutionWhere } from "../offline-migration-execution.js";
 
 export async function hasLocalNativeHarnessObligations(db: Db): Promise<boolean> {
@@ -52,7 +52,7 @@ async function prepareSource(base: string, scopeDigest: string) {
     ? entry.isFile() : names.has(entry.name) && entry.isDirectory() && !entry.isSymbolicLink());
   // Unknown extra files are preserved with an exclusive full-size reservation.
   const archiveLimit = closed && Number.isSafeInteger(bytes) && bytes <= MAX_NATIVE_HARNESS_BYTES
-    ? Math.min(MAX_NATIVE_HARNESS_OBJECT_BYTES, bytes + MAX_NATIVE_HARNESS_OBJECT_BYTES - MAX_NATIVE_HARNESS_BYTES)
+    ? nativeHarnessArchiveSizeLimit(bytes)
     : MAX_NATIVE_HARNESS_OBJECT_BYTES;
   return { root, scopeDigest, runId: state.runId as string, archiveLimit,
     sessionId: manifest.normalizedSessionId, runnerId: manifest.runnerInstanceId,