diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index a099d711f3..6d2fb52cb2 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -704,7 +704,9 @@ When effective run config changes, Paperclip may intentionally skip a saved adap Paperclip applies one process-wide scheduler to expensive host-side workspace Git enumeration, including changed-file browsing, runtime/finalization cleanliness guards, and adapter sandbox-sync snapshots. The scheduler defaults to two active scans and a bounded queue of 32. Identical scans of the same canonical worktree share one subprocess, while successful changed-file listings are cached for 10 seconds. Correctness-sensitive runtime guards bypass the result cache. -Workspace snapshots list ignored paths with `git ls-files --others --ignored --exclude-standard --directory -z` so ignored directory contents do not require a full status walk. Snapshot failures retain their typed cause instead of becoming a non-Git-folder result. During pre-provider setup, scan timeouts and queue saturation use the existing two automatic failure retries with a 30-second delay. Cancellation, output limits, and other Git errors stop with specific recovery guidance. See `doc/execution-semantics.md` for the ownership and retry-budget contract. +Workspace snapshots list ignored paths with `git ls-files --others --ignored --exclude-standard --directory -z` so ignored directory contents do not require a full status walk. The untracked-file snapshot has a 32 MiB output bound so generated trees with thousands of long filenames can be staged. It still records explicit file paths; files created after the scan do not enter that overlay. Other scan bounds stay unchanged. Snapshot failures retain their typed cause instead of becoming a non-Git-folder result. During pre-provider setup, scan timeouts and queue saturation use the existing two automatic failure retries with a 30-second delay. Cancellation, output limits, and other Git errors stop with specific recovery guidance. See `doc/execution-semantics.md` for the ownership and retry-budget contract. + +Workspace preparation resolves an existing root symlink before reading the snapshot and uses that resolved directory for the rest of the operation. Overlay staging checks the captured root identity and each selected path's ancestors before and after copying. A replaced root or a symlink in an ancestor directory stops staging before upload. A missing source file can be skipped; other source inspection errors stop staging. Selected symlink entries remain symlinks. The cache intentionally trades up to a few seconds of changed-file freshness for stable server latency. The file browser retains an explicit refresh action, does not start its query while the panel or browser tab is hidden, and presents overloads as retryable failures rather than an empty workspace. A full queue returns `503` with code `workspace_git_scan_saturated`; a scan exceeding its wall-clock limit returns `504` with code `workspace_git_scan_timeout`. Both responses include `Retry-After: 1`. diff --git a/packages/adapter-utils/src/git-workspace-sync.test.ts b/packages/adapter-utils/src/git-workspace-sync.test.ts index 5a5e054d5a..9577a10290 100644 --- a/packages/adapter-utils/src/git-workspace-sync.test.ts +++ b/packages/adapter-utils/src/git-workspace-sync.test.ts @@ -1,5 +1,5 @@ import { execFile as execFileCallback } from "node:child_process"; -import { lstat, mkdir, mkdtemp, readFile, readlink, rm, stat, symlink, writeFile } from "node:fs/promises"; +import { lstat, mkdir, mkdtemp, readFile, readlink, rename, rm, stat, symlink, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; @@ -40,7 +40,7 @@ describe("git workspace sync", () => { if (!dir) continue; await rm(dir, { recursive: true, force: true }).catch(() => undefined); } - }); + }, 30_000); // The output-limit fixture removes 40,000 files on teardown. it("delegates every host-side full-tree enumeration to the registered scheduler", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-git-scheduler-hook-")); @@ -134,6 +134,57 @@ describe("git workspace sync", () => { expect(ignoredArgs).toEqual(["ls-files", "--others", "--ignored", "--exclude-standard", "--directory", "-z"]); }); + it("snapshots a generated directory with more than 1 MiB of filenames", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-git-large-untracked-")); + cleanupDirs.push(rootDir); + const repo = await createRepo(rootDir); + const generatedDir = path.join(repo, "storybook-output"); + await mkdir(generatedDir); + const names = Array.from({ length: 5_000 }, (_, index) => `${"asset-".repeat(36)}${index}.js`); + for (let start = 0; start < names.length; start += 100) { + await Promise.all(names.slice(start, start + 100).map((name) => writeFile(path.join(generatedDir, name), ""))); + } + const raw = await runLocalGit(repo, ["ls-files", "--others", "--exclude-standard", "-z"], { + maxBuffer: 2 * 1024 * 1024, + }); + expect(Buffer.byteLength(raw.stdout)).toBeGreaterThan(1024 * 1024); + setExpensiveWorkspaceGitExecutor((input) => runLocalGit(input.localDir, [...input.args], { + timeout: input.timeout, + maxBuffer: input.maxBuffer, + })); + + const snapshot = await readGitWorkspaceSnapshot(repo); + expect(snapshot?.overlayPaths).toEqual( + names.map((name) => `storybook-output/${name}`).sort((left, right) => left.localeCompare(right)), + ); + + // A larger tree exceeds the old 8 MiB bound but fits the new 32 MiB bound. + for (let start = 5_000; start < 40_000; start += 100) { + await Promise.all(Array.from({ length: 100 }, (_, index) => writeFile( + path.join(generatedDir, `${"asset-".repeat(36)}${start + index}.js`), "", + ))); + } + const largerRaw = await runLocalGit(repo, ["ls-files", "--others", "--exclude-standard", "-z"], { + maxBuffer: 32 * 1024 * 1024, + }); + expect(Buffer.byteLength(largerRaw.stdout)).toBeGreaterThan(8 * 1024 * 1024); + const largerSnapshot = await readGitWorkspaceSnapshot(repo); + expect(largerSnapshot?.overlayPaths).toEqual( + largerRaw.stdout.split("\0").filter(Boolean).sort((left, right) => left.localeCompare(right)), + ); + + // Reuse the files with longer parent paths to exceed 32 MiB without + // creating hundreds of thousands of files solely to test the bound. + const deepParent = path.join(repo, ...Array.from({ length: 4 }, () => "nested-".repeat(30))); + await mkdir(deepParent, { recursive: true }); + await rename(generatedDir, path.join(deepParent, "storybook-output")); + expect(Buffer.byteLength(largerRaw.stdout) + 40_000 * (path.relative(repo, deepParent).length + 1)) + .toBeGreaterThan(32 * 1024 * 1024); + await expect(readGitWorkspaceSnapshot(repo)).rejects.toMatchObject({ + code: "ERR_CHILD_PROCESS_STDIO_MAXBUFFER", + }); + }, 60_000); + async function createRepo(rootDir: string): Promise { const repo = path.join(rootDir, "repo"); await mkdir(repo, { recursive: true }); diff --git a/packages/adapter-utils/src/git-workspace-sync.ts b/packages/adapter-utils/src/git-workspace-sync.ts index f3ade95f7c..46d461d397 100644 --- a/packages/adapter-utils/src/git-workspace-sync.ts +++ b/packages/adapter-utils/src/git-workspace-sync.ts @@ -201,9 +201,12 @@ export async function readGitWorkspaceSnapshot(localDir: string, includeReposito timeout: 10_000, maxBuffer: 1024 * 1024, }), + // A generated output tree can exceed 1 MiB of filenames with only a few + // thousand files. Keep the explicit file snapshot (and a finite bound): + // collapsing directories would let later files enter the staging copy. runExpensiveWorkspaceGit(localDir, ["ls-files", "--others", "--exclude-standard", "-z"], "adapter_sync.untracked_files", { timeout: 10_000, - maxBuffer: 1024 * 1024, + maxBuffer: 32 * 1024 * 1024, }), runExpensiveWorkspaceGit(localDir, ["diff", "--name-only", "-z", "--diff-filter=D", "HEAD", "--"], "adapter_sync.deleted_files", { timeout: 10_000, diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts index e04d5a4ee2..b75148914d 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts @@ -847,7 +847,7 @@ describe("sandbox managed runtime", () => { await git(sourceRepoDir, ["checkout", "-b", "main"]); await git(sourceRepoDir, ["config", "user.name", "Paperclip Test"]); await git(sourceRepoDir, ["config", "user.email", "test@paperclip.dev"]); - await writeFile(path.join(sourceRepoDir, ".gitignore"), "node_modules/\n", "utf8"); + await writeFile(path.join(sourceRepoDir, ".gitignore"), "node_modules/\n*.secret\n", "utf8"); await writeFile(path.join(sourceRepoDir, "tracked.txt"), "base\n", "utf8"); await writeFile(path.join(sourceRepoDir, "clean.txt"), "from git\n", "utf8"); await writeFile(path.join(sourceRepoDir, "deleted.txt"), "delete me\n", "utf8"); @@ -859,6 +859,9 @@ describe("sandbox managed runtime", () => { await mkdir(path.join(localWorkspaceDir, "node_modules"), { recursive: true }); await writeFile(path.join(localWorkspaceDir, "tracked.txt"), "dirty local\n", "utf8"); await writeFile(path.join(localWorkspaceDir, "untracked.txt"), "from local\n", "utf8"); + await mkdir(path.join(localWorkspaceDir, "drafts")); + await writeFile(path.join(localWorkspaceDir, "drafts", "report.md"), "draft\n"); + await symlink("report.md", path.join(localWorkspaceDir, "drafts", "report-link.md")); await writeFile(path.join(localWorkspaceDir, "node_modules", "cache.bin"), "do not upload\n", "utf8"); await rm(path.join(localWorkspaceDir, "deleted.txt")); @@ -914,6 +917,11 @@ describe("sandbox managed runtime", () => { workspaceLocalDir: localWorkspaceDir, onRuntimeProgress: async (status) => { runtimeStatuses.push({ phase: status.phase, message: status.message }); + if (status.phase === "config_sync") { + // These files appear after the Git snapshot, before the overlay copy. + await writeFile(path.join(localWorkspaceDir, "drafts", "late.secret"), "private\n"); + await writeFile(path.join(localWorkspaceDir, "drafts", "late.txt"), "later work\n"); + } }, }); @@ -936,6 +944,10 @@ describe("sandbox managed runtime", () => { expect(workspaceMembers.some((entry) => entry === ".git" || entry.startsWith(".git/"))).toBe(false); expect(workspaceMembers).toContain("tracked.txt"); expect(workspaceMembers).toContain("untracked.txt"); + expect(workspaceMembers).toContain("drafts/report.md"); + expect(await readlink(path.join(remoteWorkspaceDir, "drafts", "report-link.md"))).toBe("report.md"); + expect(workspaceMembers).not.toContain("drafts/late.secret"); + expect(workspaceMembers).not.toContain("drafts/late.txt"); expect(workspaceMembers).not.toContain("clean.txt"); expect(workspaceMembers.some((entry) => entry === "node_modules" || entry.startsWith("node_modules/"))).toBe(false); @@ -1199,6 +1211,104 @@ describe("sandbox managed runtime", () => { expect(downloadMembers.some((entry) => entry.includes("/node_modules/") || entry.endsWith("/node_modules"))).toBe(false); }); + it.each(["symlink", "root_symlink", "root_alias", "root_alias_retarget", "case_alias", "EACCES", "EIO", "ENOENT"])("handles an overlay source changed after the snapshot: %s", async (change) => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-sandbox-overlay-source-")); + cleanupDirs.push(rootDir); + const workspaceLocalDir = path.join(rootDir, "workspace"); + await initGitRepo(workspaceLocalDir); + const draftsDir = path.join(workspaceLocalDir, "drafts"); + const selectedPath = path.join(change === "root_symlink" ? workspaceLocalDir : draftsDir, "report.md"); + await mkdir(draftsDir); + await writeFile(selectedPath, "selected work\n"); + const workspaceInputDir = change.startsWith("root_alias") ? path.join(rootDir, "workspace-alias") : workspaceLocalDir; + if (change.startsWith("root_alias")) await symlink(workspaceLocalDir, workspaceInputDir); + const outsideDir = path.join(rootDir, "outside"); + await mkdir(outsideDir); + await writeFile(path.join(outsideDir, "report.md"), "private outside content\n"); + await mkdir(path.join(outsideDir, "drafts")); + await writeFile(path.join(outsideDir, "drafts", "report.md"), "private outside content\n"); + const syncIn = vi.fn(async (operations: SandboxSyncOperation[]) => { + if (change.startsWith("root_alias")) { + const overlay = operations.flatMap((operation) => operation.files) + .find((file) => path.basename(file.targetPath) === "workspace-upload.tar"); + expect(overlay).toBeDefined(); + const contents = await execFile("tar", ["-xOf", overlay!.sourcePath, "drafts/report.md"]); + expect(contents.stdout).toBe("selected work\n"); + } + return { operations: [] }; + }); + const client: SandboxManagedRuntimeClient = { + makeDir: async () => {}, + writeFile: async () => {}, + readFile: async () => new ArrayBuffer(0), + listFiles: async () => [], + remove: async () => {}, + run: async () => {}, + syncIn, + }; + const realLstat = fsPromises.lstat.bind(fsPromises); + const realRealpath = fsPromises.realpath.bind(fsPromises); + const failure = Object.assign(new Error(`Cannot inspect overlay: ${change}`), { code: change }); + let statSpy: ReturnType | undefined; + let realpathSpy: ReturnType | undefined; + try { + const preparing = prepareSandboxManagedRuntime({ + spec: { + transport: "sandbox", + provider: "test", + sandboxId: "sandbox-1", + remoteCwd: path.join(rootDir, "remote"), + timeoutMs: 30_000, + apiKey: null, + }, + adapterKey: "test-adapter", + client, + workspaceLocalDir: workspaceInputDir, + onRuntimeProgress: async (status) => { + if (status.phase !== "config_sync") return; + if (change === "root_alias") return; + if (change === "root_alias_retarget") { + await rm(workspaceInputDir); + await symlink(outsideDir, workspaceInputDir); + return; + } + if (change === "symlink") { + await rm(draftsDir, { recursive: true }); + await symlink(outsideDir, draftsDir); + } else if (change === "ENOENT") { + await rm(selectedPath); + } else if (change === "case_alias") { + // Model a case-insensitive filesystem: Git's indexed spelling + // still resolves, while realpath reports the directory's new case. + realpathSpy = vi.spyOn(fsPromises, "realpath").mockImplementation((async (...args: Parameters) => { + if (args[0] === draftsDir) return path.join(workspaceLocalDir, "Drafts"); + return realRealpath(...args); + }) as typeof fsPromises.realpath); + } else if (change === "root_symlink") { + // The root was captured with the snapshot, before config_sync. + await fsPromises.rename(workspaceLocalDir, path.join(rootDir, "original-workspace")); + await symlink(outsideDir, workspaceLocalDir); + } else { + statSpy = vi.spyOn(fsPromises, "lstat").mockImplementation((async (...args: Parameters) => { + if (args[0] === selectedPath) throw failure; + return realLstat(...args); + }) as typeof fsPromises.lstat); + } + }, + }); + if (change === "ENOENT" || change === "case_alias" || change.startsWith("root_alias")) { + await preparing; + expect(syncIn).toHaveBeenCalledOnce(); + } else { + await expect(preparing).rejects.toThrow(change.endsWith("symlink") ? /overlay.*directory/i : failure.message); + expect(syncIn).not.toHaveBeenCalled(); + } + } finally { + statSpy?.mockRestore(); + realpathSpy?.mockRestore(); + } + }); + it("excludes an anchor-workspace ignored file whose name has leading and trailing whitespace from the staged tree", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-sandbox-ignored-whitespace-")); cleanupDirs.push(rootDir); diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.ts b/packages/adapter-utils/src/sandbox-managed-runtime.ts index 7fd3ab0d97..e7ebbfee5e 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.ts @@ -931,18 +931,66 @@ export async function mirrorDirectory( } } -async function copySelectedWorkspaceEntries(input: { +interface WorkspaceSourceRoot { sourceDir: string; + dev: number; + ino: number; +} + +async function captureWorkspaceSourceRoot(localDir: string): Promise { + const sourceDir = await fs.realpath(localDir); + const stats = await fs.lstat(sourceDir); + if (!stats.isDirectory()) throw new Error("Workspace overlay root is not a directory"); + return { sourceDir, dev: stats.dev, ino: stats.ino }; +} + +async function copySelectedWorkspaceEntries(input: { + sourceRoot: WorkspaceSourceRoot; targetDir: string; relativePaths: string[]; exclude: string[]; }): Promise { await fs.mkdir(input.targetDir, { recursive: true }); + const { sourceDir, dev, ino } = input.sourceRoot; + const assertSourceRoot = async () => { + const current = await fs.lstat(sourceDir); + if (!current.isDirectory() || current.dev !== dev || current.ino !== ino) { + throw new Error("Workspace overlay root directory changed during staging"); + } + }; + await assertSourceRoot(); for (const relative of input.relativePaths) { if (shouldExcludePath(relative, input.exclude)) continue; - const sourceStats = await fs.lstat(path.join(input.sourceDir, relative)).catch(() => null); - if (!sourceStats) continue; - await copyWorkspaceEntry(input.sourceDir, input.targetDir, relative); + const sourcePath = path.join(sourceDir, relative); + const parentSegments = path.relative(sourceDir, path.dirname(sourcePath)).split(path.sep).filter(Boolean); + const assertParentDirectory = async () => { + // Git selected this path before staging. A replaced ancestor must not + // redirect the copy through a symlink, even to another workspace folder. + // Inspect types instead of comparing realpath spelling: case-insensitive + // filesystems can resolve Git's indexed casing to a renamed directory. + let parentPath = sourceDir; + for (const segment of parentSegments) { + if (segment === "..") throw new Error(`Workspace overlay directory escapes its root: ${relative}`); + parentPath = path.join(parentPath, segment); + if (!(await fs.lstat(parentPath)).isDirectory()) { + throw new Error(`Workspace overlay ancestor is not a directory: ${relative}`); + } + } + }; + // Include root-level entries, and do not treat a missing root as an + // ordinary source file that disappeared after the snapshot. + await assertSourceRoot(); + try { + await assertParentDirectory(); + await fs.lstat(sourcePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; + throw error; + } + await copyWorkspaceEntry(sourceDir, input.targetDir, relative); + // Do not upload the staged tree if an ancestor changed during the copy. + await assertSourceRoot(); + await assertParentDirectory(); } } @@ -1132,6 +1180,12 @@ export async function prepareSandboxManagedRuntime(input: { const runStepSpan = (name: string, work: () => Promise): Promise => input.runtimeSpan ? input.runtimeSpan(name, work) : work(); + // Resolve an existing workspace alias once, before reading its snapshot. + // All subsequent work uses that root, so retargeting the alias cannot select + // another repository. Staging also verifies the captured directory identity. + const workspaceRoot = syncWorkspace ? await captureWorkspaceSourceRoot(input.workspaceLocalDir) : null; + if (workspaceRoot) input = { ...input, workspaceLocalDir: workspaceRoot.sourceDir }; + // The git enumeration (`git status --ignored`, the HEAD diffs, `ls-files`). // It reads git's own bookkeeping to decide what to include/exclude, so it is // usually fast, but on a large working tree the `--ignored` walk is not free. @@ -1404,7 +1458,7 @@ export async function prepareSandboxManagedRuntime(input: { : input.workspaceLocalDir; if (gitSnapshot) { await copySelectedWorkspaceEntries({ - sourceDir: input.workspaceLocalDir, + sourceRoot: workspaceRoot!, targetDir: workspaceArchiveDir, relativePaths: gitSnapshot.overlayPaths, exclude: workspaceArchiveExclude,