From ce0eff7d1fe670f2ef3343178c144b1e4fa9b2ce Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:21:47 -0500 Subject: [PATCH] perf: reduce fresh workspace clone and sandbox staging waits (#16487) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip manages AI agents and their work. > - Each run needs a private workspace and safe persistence. > - Fresh isolated repository sources download upstream Git history again. > - Sandbox staging also verifies an optional seed cache before it uploads the workspace. > - That verification holds a company lock during a full extraction and hash scan. > - This pull request reuses pristine Git objects through an authenticated remote clone. > - It also overlaps seed verification with upload and moves verification outside the publication lock. > - The benefit is less repeated transfer and less serial preparation work, with the same task isolation and recovery contract. ## Linked Issues or Issue Description **What existing behavior does this improve?** Fresh task-private repository preparation and inbound sandbox workspace staging. **Current behavior** Each isolated remote source clones the full upstream history. Optional seed verification then extracts and hashes another full tree before upload. One workspace's verification can hold the company cache lock while other workspaces wait. **Proposed behavior** Use a company and credential scoped cache of pristine upstream objects for GitHub HTTPS sources. Each task still clones from the current remote. It disconnects from the cache before publication. Run seed verification alongside upload. Hold the company lock only for final capacity checks and publication. **Reason and benefit** These operations lie on the workspace preparation path. Removing repeated Git pack transfer and serial verification can reduce startup time. Local measurements below cover these boundaries. They do not establish hosted agent latency. Related: #16425, #16454, #16398, #16461, #16468, #16473. The cache inspection and durable folder proposal in #13376 covers separate product work. ## What Changed - Add a pristine upstream Git object cache for fresh isolated GitHub HTTPS sources. Key it by company, repository and current credential fingerprint. Do not store credentials. - Keep online clone authorization and current remote refs. Use `--reference-if-able --dissociate` so task objects remain independent. A reference failure retries once online without the cache and resolves credentials again. - Cap cache storage at 64 entries and 4 GiB per company, with 2 GiB per entry. Skip contended publication. Bound priming by a cooperative five-second budget and a 4,096-entry scan allowance. Keep the existing free-space reserve. - Overlap optional seed verification and required workspace upload. Settle both before returning or deleting temporary files. Recheck quotas and company directory identity under the publication lock. - Keep the independent fresh upstream reproducibility proof, task commits, dirty source copies, ignored files, nested repositories and durable recovery behavior. - Suppress optional Git server-info generation during dissociation so generated storage passes the existing safe-expiration checks. This uses [the supported Git setting](https://git-scm.com/docs/git-config/2.43.0#Documentation/git-config.txt-repackupdateServerInfo). - Add bounded timing phases, regression coverage and two repeatable local benchmarks. No first-party Telemetry or dependency changes. ## Verification - `pnpm -r typecheck`: passed. - `pnpm build`: passed. - Affected adapter suites: 240 tests passed. Native workspace persistence/history and tracing: 33 tests passed. Server checkout/isolation/tracing batch: 53 tests passed. Final object-cache authority, path and capacity suite: 12 tests passed. These batches overlap. - Cache and expiration/restoration lifecycle batch: 34 tests passed. A direct generated-storage check covers cached clones. - Real HTTP Git fixture: a matching cache does not bypass revoked remote access. Other fixtures cover current refs/tags, hidden cached commits, independent objects after cache removal, corruption fallback, credential/company scope, quota limits and publication contention. - Initial restricted runs hit SQLite write and Unix socket permission failures. The adapter batch passed with disposable writable instance state and socket access. The first typecheck rejected the test use of `Promise.withResolvers`; typed deferred gates now use the supported library contract. - Repository benchmark on final head `9047368bf3b7fa2061e00d698b8b51cf4ae1da32`: five alternating samples, real local `file://` Git transport, synthetic 32 MiB history, no injected latency. Median 1143.8 ms to 240.5 ms; ranges 1085.7–1486.6 ms and 188.0–254.8 ms. Received Git pack bytes fell from 33,569,788 to zero. Initial cache priming took 1208.4 ms and is reported separately. - Sandbox benchmark, five alternating samples per mode on 10,000 files with real packing, extraction and verification: with a controlled 3-second transfer delay, median 9.450 s to 7.620 s (ranges 9.280–12.115 s and 7.296–7.983 s). With no injected delay, median 6.808 s to 6.450 s (ranges 6.255–7.029 s and 5.709–6.902 s). The no-delay ranges overlap; local copying competes with verification for disk. Both modes verify final tree contents, ignored-file exclusion and Git HEAD. These are synthetic local fixtures, not hosted agent runs. - Reproduce with `pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-repository-object-cache.ts` and `pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-workspace-staging.ts --baseline-ref=307eb16c0c --transfer-delay-ms=3000`. - Prior head `af766bc820` received Greptile 5/5 with no findings. Its E2E shard 7 failed in `plan-task-fixture` cleanup: company deletion raced an `activity_log` foreign key. That fixture has no provider run or Git workspace. The failure is retained here; final-head CI must pass. The corrected corruption fixture now forces a pack and verifies a real damaged archive. Current head `9047368bf3b7fa2061e00d698b8b51cf4ae1da32` has Greptile 5/5 with no review threads or findings. Final head has 55 passing checks and two optional Storybook skips. [The first workspace-shard attempt](https://github.com/paperclipai/paperclip/actions/runs/38802441857/job/116655485350) passed all 1,778 cases, then reported a late `EPIPE` from unchanged `server-utils.ts` while the ACP reuse test was active. That case mocks workspace preparation and passed locally in isolation. [Its single retry on the same commit](https://github.com/paperclipai/paperclip/actions/runs/38802441857/job/116659728034) passed, as did the dependent verification gate. The earlier E2E cleanup fixture passed on this final head. Both initial failures remain recorded here. A full local suite was not duplicated; the affected boundaries were run locally and the complete suite runs in CI. ## Risks - First cache population adds local copying. Cold clones, credential rotation, capacity exhaustion and a corrupt cache retain the online clone path. No eviction is added. Cached data is bounded, but many companies can each consume their allowance. - Individual filesystem calls and cleanup can exceed the cooperative priming budget. Failed reference clones make one additional online attempt. - Parallel verification and upload compete for disk and CPU. The local no-delay benchmark shows a smaller gain than a slow-transfer case. Verification still checks contents before cache publication, and required upload failures remain failures. - Local fixtures do not prove hosted throughput or the 15–30-second end-to-end target. The fresh checkout still writes its working tree, and model startup, copyback and other preparation costs remain. ## Model Used OpenAI Codex, GPT-6 family. The exact serving model ID and context window are not exposed in this session. Used reasoning, source editing, shell tools and test execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes: #` / `Refs: #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change and contains no internal task identifiers - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- doc/DEVELOPING.md | 58 ++++++ doc/observability.md | 2 +- doc/plans/2026-10-09-task-workspaces.md | 2 +- .../src/sandbox-managed-runtime.ts | 59 +++--- .../src/sandbox-managed-seed.test.ts | 51 ++++- .../src/workspace-restore-merge.ts | 10 + .../src/workspace-seed-cache.test.ts | 51 +++++ .../adapter-utils/src/workspace-seed-cache.ts | 16 +- scripts/benchmark-repository-object-cache.ts | 62 ++++++ scripts/benchmark-workspace-staging.ts | 116 +++++++++++ .../__tests__/repository-object-cache.test.ts | 183 ++++++++++++++++++ .../services/execution-preparation-timing.ts | 1 + .../isolated-remote-workspace-source.ts | 4 +- .../services/managed-repository-checkout.ts | 20 +- .../src/services/repository-object-cache.ts | 111 +++++++++++ 15 files changed, 708 insertions(+), 38 deletions(-) create mode 100644 scripts/benchmark-repository-object-cache.ts create mode 100644 scripts/benchmark-workspace-staging.ts create mode 100644 server/src/__tests__/repository-object-cache.test.ts create mode 100644 server/src/services/repository-object-cache.ts diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index bd7d1acdfb5..019d752eb2a 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -686,6 +686,35 @@ per-query results and latency measurements; scale measurements are opt-in. ### Repository workspace admission benchmark +Measure a fresh checkout with and without the repository object cache: + +```sh +pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-repository-object-cache.ts +``` + +This uses a synthetic 32 MiB repository and the real `file://` Git transport. +It reports cache priming separately, then alternates five fresh checkouts per +mode. Git's pack trace records received bytes. Each checkout has independent +objects and passes `git fsck`. There is no injected latency or hosted network +measurement. + +Measure cold sandbox staging with a disposable 10,000-file Git repository: + +```sh +pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-workspace-staging.ts --baseline-ref=origin/master +pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-workspace-staging.ts --baseline-ref=origin/master --transfer-delay-ms=3000 +``` + +The benchmark alternates the baseline and current implementations. Both pack, +verify, copy and extract real archives, then compare the resulting tree and Git +HEAD. It retains dirty, deleted and untracked-file semantics and excludes ignored +files. The transport is a local copy. The optional delay models a slow transfer; +it does not measure hosted network or agent latency. The report contains every +sample, whole-staging wall time, archive bytes and phase durations. Seed +verification and upload can overlap, so do not sum their durations. The script +temporarily imports baseline source beside the current source and removes it on +exit. Run without other repository edits or heavy I/O for a useful comparison. + Measure the real repository preparation service with a disposable embedded PostgreSQL database and synthetic history, without invoking an agent: @@ -1104,6 +1133,35 @@ When effective run config changes, Paperclip may intentionally skip a saved adap An explicit `reuse_existing` Git workspace keeps its recorded source repository, including after its project-workspace row is deleted or the project primary changes. The server verifies the original company, project, repository origin, and worktree registration before reuse. It does not attach a deleted association to the new primary. A conflicting explicit project-workspace selection, missing source identity, or an unverified legacy local checkout blocks the run. Restore the original source. Before intentionally clearing the existing-workspace binding to choose a new repository, review and preserve its retained work. Missing worktrees may be reconstructed only from the verified original source; current project defaults are not a fallback. +### Isolated repository object cache + +Fresh task-private GitHub HTTPS sources use a pristine upstream object cache at +`/repository-object-cache//`. The scope +includes the repository URL and current credential fingerprint. Credentials +are not stored in the cache. Other remote providers retain the ordinary clone +path. Existing task sources and local dirty source copies keep their current +behavior. + +Every new checkout still contacts the remote with current Git credentials. +`git clone --reference-if-able --dissociate` reuses objects but selects the +remote's current refs and creates independent storage. An unavailable or denied +remote cannot fall back to cached files. A failed reference clone retries once +without the cache and resolves credentials again. Task commits and dirty files +never populate this cache. Reproducibility checks continue to use a fresh upstream object store for +their independent proof; they do not use this cache. + +Cache publication is optional and never waits for another publisher. It keeps +at most 64 entries and 4 GiB per company, with 2 GiB per entry. Temporary cache +construction can use one additional entry's space; the existing manifest free +space reserve applies. Priming uses a five-second cooperative work budget and +a 4,096-entry scan allowance; individual filesystem calls and cleanup can +exceed that time. Existing entries and abandoned pending entries count +against capacity. A full, corrupt or unavailable cache leaves ordinary cloning +available. No cache eviction is performed. Credential rotation selects another +scope and can reduce reuse after capacity is exhausted. There is no link from +an admitted task checkout back to this cache, so cache removal cannot remove +task Git objects. + ## Workspace Git Scan Protection Paperclip applies one process-wide scheduler to expensive host-side workspace Git enumeration, including changed-file browsing, runtime/finalization cleanliness guards, and adapter sandbox-sync snapshots. The scheduler defaults to two active scans and a bounded queue of 32. Identical buffered scans of the same canonical worktree share one subprocess, while successful changed-file listings are cached for 10 seconds. Streaming snapshot scans have caller-owned sinks, so they use separate jobs in the same queue and are never cached or coalesced. Correctness-sensitive runtime guards bypass the result cache. diff --git a/doc/observability.md b/doc/observability.md index 6068bc13022..31f5f7335c8 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -289,7 +289,7 @@ attempt or owning phase: | `workspace.sources.*` | Source inventory queries, canonicalization, one filesystem alias scan, repository scan, and configured project scan. | | `workspace.readers`, `workspace.readers.*`, `workspace.access.*` | Retained-reader groups, identity, trust, project and source authorization, privacy and inventory reads. | | `workspace.provenance`, `workspace.notification`, `workspace.write` | Preserved ownership, notification statements, and record writes. | -| `workspace.materialize.*`, `workspace.checkout.*` | Private source-directory setup and identity verification, existing checkout lookup, Git credential resolution, clone, requested-ref checkout, and atomic publication. The enclosing `heartbeat.prepare.workspace_source_materialize` includes the complete source-materialization path. | +| `workspace.materialize.*`, `workspace.checkout.*` | Private source-directory setup and identity verification, existing checkout lookup, Git credential resolution, clone, optional pristine-object cache lookup/publication (`cache_lookup`, `cache_publish`), requested-ref checkout, and atomic publication. The enclosing `heartbeat.prepare.workspace_source_materialize` includes the complete source-materialization path. | | `runtime_selection.*` | Completion contracts, session/recovery reads, profiles, interaction materialization, backend discovery, bootstrap, run/coordinator writes, transaction entry/settlement, and workspace synchronization. | | `runtime_context.instructions`, `.skills`, `.mcp` | The three concurrent runtime-context branches. Their durations overlap. | | `runtime_context.asset_*` | Source reads, manifest hashing, cache lookup, cold materialization, and exact file verification on cold and warm paths. | diff --git a/doc/plans/2026-10-09-task-workspaces.md b/doc/plans/2026-10-09-task-workspaces.md index fdfb1e2dad7..374fb303284 100644 --- a/doc/plans/2026-10-09-task-workspaces.md +++ b/doc/plans/2026-10-09-task-workspaces.md @@ -71,7 +71,7 @@ Sparse filesystem checkpoints use disk-backed full manifests, changed bytes/dele A returned sparse snapshot must describe a coherent tree before any host mutation. Every nested entry requires an immediate parent marked as a directory in that same returned snapshot, including baseline-unchanged files. Checking every directory also verifies recursive ancestry. Top-level entries use the implicit root. Invalid trees fail without overwriting retained host files and keep recovery evidence. The check uses the existing indexed manifest interface; it adds no host-path scans or unbounded in-memory inventory. -Seed recovery deliberately uses **no delta chain** in this implementation. An unchanged generation reuses its verified content-addressed immutable full seed; a changed generation creates a fresh full seed. Sparse output transfer is separate from seed recovery. This revises the sketch's 16-delta compaction proposal: composing recovery generations would add a new failure surface before the simpler cache has been qualified. Existing references retain old seeds; no automatic eviction/deletion policy is introduced. Optional cache publication is capped at 16 generations/1 GiB per workspace and 256 generations/256 workspace directories/8 GiB per company. A company-scoped filesystem lock serializes quota checks and publication across workspace roots. Archive and receipt bytes, corrupt entries and abandoned pending generations count against capacity; publication also preserves the manifest storage free-space reserve (256 MiB by default), accounting for a full-copy fallback. Saturation or cache failure skips publication and keeps the already-written per-run recovery seed. Existing over-capacity caches stop growing without deleting admitted evidence. This deliberately trades cache hit rate after saturation for bounded additional storage and unchanged recovery semantics; no global cross-company quota is added. Changed input generations still pay full snapshot creation. New cache entries are reconstructed and checked against the admitted baseline before publication, so concurrent writes cannot label an archive as the wrong generation; this adds cold verification work. A corrupted cache entry remains a miss and falls back to a fresh seed; automatic cache repair and garbage collection are deferred. The existing Git seed contract captures the selected branch/HEAD, not unrelated refs. +Seed recovery deliberately uses **no delta chain** in this implementation. An unchanged generation reuses its verified content-addressed immutable full seed; a changed generation creates a fresh full seed. Sparse output transfer is separate from seed recovery. This revises the sketch's 16-delta compaction proposal: composing recovery generations would add a new failure surface before the simpler cache has been qualified. Existing references retain old seeds; no automatic eviction/deletion policy is introduced. Optional cache publication is capped at 16 generations/1 GiB per workspace and 256 generations/256 workspace directories/8 GiB per company. A company-scoped filesystem lock serializes final quota checks and publication across workspace roots. Optional seed reconstruction and baseline verification run outside that lock, alongside inbound upload. Preparation settles both operations before returning or removing temporary archives. Publication rechecks capacity and directory identity under the lock after verification; a competing publication can consume the remaining capacity and cause this optional cache write to be skipped. Archive and receipt bytes, corrupt entries and abandoned pending generations count against capacity; publication also preserves the manifest storage free-space reserve (256 MiB by default), accounting for a full-copy fallback. Saturation or cache failure skips publication and keeps the already-written per-run recovery seed. Existing over-capacity caches stop growing without deleting admitted evidence. This deliberately trades cache hit rate after saturation for bounded additional storage and unchanged recovery semantics; no global cross-company quota is added. Changed input generations still pay full snapshot creation. New cache entries are reconstructed and checked against the admitted baseline before publication, so concurrent writes cannot label an archive as the wrong generation; this adds cold verification work. A corrupted cache entry remains a miss and falls back to a fresh seed; automatic cache repair and garbage collection are deferred. The existing Git seed contract captures the selected branch/HEAD, not unrelated refs. Ordinary settlement remains host-durable before successful completion. Preserve export-only recovery and exact-source lease retention. Preserve the specific unsafe-export omission exception and its run-log-only diagnostic. Do not change global merge conflict semantics as a side effect. diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.ts b/packages/adapter-utils/src/sandbox-managed-runtime.ts index 7458f3953e4..abbb34aca46 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.ts @@ -1640,38 +1640,43 @@ export async function prepareSandboxManagedRuntime(input: { workspaceUploadBytes += (await fs.stat(workspaceTarPath)).size; }); - if (seedGeneration && !cachedSeed && input.workspaceDurableSeed) { - const seed = input.workspaceDurableSeed; - await runStepSpan("workspace.seed_publish", () => publishWorkspaceSeedGeneration(input.workspaceSeedCacheDirectory!, seedGeneration, - { ...seed, gitArchivePath: gitSnapshot || repositories.length ? seed.gitArchivePath : null }, { - companyDirectory: input.workspaceSeedCacheCompanyDirectory, - verify: () => runStepSpan("seed.verify", async () => { - const verifiedRoot = path.join(tempDir, "verify-seed"); - try { - if (gitSnapshot || repositories.length) await runStepSpan("workspace.seed_extract", () => extractTarballToDirectory({ archivePath: path.join(tempDir, "git-workspace.tar"), localDir: verifiedRoot })); - await runStepSpan("workspace.seed_extract", () => extractTarballToDirectory({ archivePath: path.join(tempDir, "workspace.tar"), localDir: verifiedRoot, exclude: repositoryMetadataExcludes })); - for (const relative of workspacePaths(deletedPaths)) { - let parent = verifiedRoot; - for (const segment of relative.split("/").slice(0, -1)) { - parent = path.join(parent, segment); - const stat = await fs.lstat(parent).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); - if (stat && !stat.isDirectory()) return false; + const publishSeed = async () => { + if (seedGeneration && !cachedSeed && input.workspaceDurableSeed) { + const seed = input.workspaceDurableSeed; + await runStepSpan("workspace.seed_publish", () => publishWorkspaceSeedGeneration(input.workspaceSeedCacheDirectory!, seedGeneration, + { ...seed, gitArchivePath: gitSnapshot || repositories.length ? seed.gitArchivePath : null }, { + companyDirectory: input.workspaceSeedCacheCompanyDirectory, + verify: () => runStepSpan("seed.verify", async () => { + const verifiedRoot = path.join(tempDir, "verify-seed"); + try { + if (gitSnapshot || repositories.length) await runStepSpan("workspace.seed_extract", () => extractTarballToDirectory({ archivePath: path.join(tempDir, "git-workspace.tar"), localDir: verifiedRoot })); + await runStepSpan("workspace.seed_extract", () => extractTarballToDirectory({ archivePath: path.join(tempDir, "workspace.tar"), localDir: verifiedRoot, exclude: repositoryMetadataExcludes })); + for (const relative of workspacePaths(deletedPaths)) { + let parent = verifiedRoot; + for (const segment of relative.split("/").slice(0, -1)) { + parent = path.join(parent, segment); + const stat = await fs.lstat(parent).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); + if (stat && !stat.isDirectory()) return false; + } + await fs.rm(path.join(verifiedRoot, relative), { recursive: true, force: true }); } - await fs.rm(path.join(verifiedRoot, relative), { recursive: true, force: true }); - } - const verified = await runStepSpan("workspace.seed_snapshot", () => captureWorkspaceSnapshot(verifiedRoot, { exclude: baselineSnapshot!.exclude, ignoredPaths: baselineSnapshot!.ignoredPaths, diskBacked: true })); - try { return directorySnapshotSha256(verified) === directorySnapshotSha256(baselineSnapshot!); } - finally { await disposeDirectorySnapshot(verified); } - } finally { await runStepSpan("workspace.seed_cleanup", () => fs.rm(verifiedRoot, { recursive: true, force: true })); } - }), - })); + const verified = await runStepSpan("workspace.seed_snapshot", () => captureWorkspaceSnapshot(verifiedRoot, { exclude: baselineSnapshot!.exclude, ignoredPaths: baselineSnapshot!.ignoredPaths, diskBacked: true })); + try { return directorySnapshotSha256(verified) === directorySnapshotSha256(baselineSnapshot!); } + finally { await disposeDirectorySnapshot(verified); } + } finally { await runStepSpan("workspace.seed_cleanup", () => fs.rm(verifiedRoot, { recursive: true, force: true })); } + }), + })); + } + }; + if (!stageWorkspace) { + await publishSeed(); + return; } - if (!stageWorkspace) return; // One confined `syncIn` for the whole merged workspace file set. The confine // guard covers every mapping BEFORE any bytes upload (fail-closed): a source // or target escape in EITHER tar mapping stops the upload of both. - await stageConfinedSyncIn({ + const staging = stageConfinedSyncIn({ files: workspaceFiles, postUploadCommands: workspacePostUploadCommands, sourceRoots: [tempDir], @@ -1680,6 +1685,8 @@ export async function prepareSandboxManagedRuntime(input: { statusPhase: "config_sync", progressBytes: workspaceUploadBytes, }); + const [staged] = await Promise.allSettled([staging, publishSeed()]); + if (staged.status === "rejected") throw staged.reason; }), ); } diff --git a/packages/adapter-utils/src/sandbox-managed-seed.test.ts b/packages/adapter-utils/src/sandbox-managed-seed.test.ts index e54bdc7d8e7..4e563f5debb 100644 --- a/packages/adapter-utils/src/sandbox-managed-seed.test.ts +++ b/packages/adapter-utils/src/sandbox-managed-seed.test.ts @@ -8,6 +8,12 @@ import { createTarballFromDirectory, prepareSandboxManagedRuntime, type SandboxS import { captureDirectorySnapshot } from "./workspace-restore-merge.js"; const execute = promisify(execFile); +function deferred() { + let resolve!: (value: T | PromiseLike) => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} + const roots: string[] = []; afterEach(async () => { vi.unstubAllEnvs(); @@ -48,9 +54,6 @@ it.each([undefined, "all"] as const)("keeps nested Git metadata sanitized in col for (const file of operation.files) { await fs.mkdir(path.dirname(file.targetPath), { recursive: true }); await fs.cp(file.sourcePath, file.targetPath, { recursive: true, verbatimSymlinks: true }); - if (file.sourcePath.endsWith("/workspace.tar")) { - await expect(fs.stat(path.join(path.dirname(file.sourcePath), "verify-seed"))).rejects.toMatchObject({ code: "ENOENT" }); - } } for (const command of operation.postUploadCommands ?? []) await run(command.command); } @@ -111,3 +114,45 @@ it.each([undefined, "all"] as const)("keeps nested Git metadata sanitized in col } await recovered.cleanupWorkspaceSnapshot(); await cached.cleanupWorkspaceSnapshot(); }, 30000); + +it.each([false, true])("overlaps seed verification and upload and drains both before %s failure settles", async (failUpload) => { + const temp = await fs.mkdtemp(path.join(os.tmpdir(), "managed-seed-overlap-")); roots.push(temp); + vi.stubEnv("PAPERCLIP_HOME", path.join(temp, "home")); + const host = path.join(temp, "host"); await fs.mkdir(host); + await fs.writeFile(path.join(host, "work.txt"), "original\n"); + const entered = deferred(); + const release = deferred(); + let verificationFinished = false, settled = false; + const preparing = prepareSandboxManagedRuntime({ + spec: { provider: "test", sandboxId: "test", remoteCwd: path.join(temp, "remote"), timeoutMs: 30000, apiKey: null, transport: "sandbox" }, + adapterKey: "test", workspaceLocalDir: host, workspaceFileMode: "all", + workspaceSeedCacheDirectory: path.join(temp, "cache"), + workspaceDurableSeed: { workspaceArchivePath: path.join(temp, "seed.tar") }, + runtimeSpan: async (name, work) => { + if (name !== "seed.verify") return work(); + entered.resolve(); + await release.promise; + try { return await work(); } finally { verificationFinished = true; } + }, + client: { + makeDir: async () => {}, run: async () => {}, writeFile: async () => {}, + readFile: async () => Buffer.alloc(0), listFiles: async () => [], remove: async () => {}, + syncIn: async () => { + await entered.promise; + expect(verificationFinished).toBe(false); + if (failUpload) throw new Error("upload failed"); + return { operations: [] }; + }, + }, + }); + const result = preparing.then(runtime => { settled = true; return { runtime }; }, error => { settled = true; return { error }; }); + try { + await entered.promise; + await new Promise(resolve => setImmediate(resolve)); + expect(settled).toBe(false); + } finally { release.resolve(); } + const outcome = await result; + expect(verificationFinished).toBe(true); + if ("error" in outcome) { expect(failUpload).toBe(true); expect(outcome.error.message).toBe("upload failed"); } + else { expect(failUpload).toBe(false); await outcome.runtime.cleanupWorkspaceSnapshot(); } +}); diff --git a/packages/adapter-utils/src/workspace-restore-merge.ts b/packages/adapter-utils/src/workspace-restore-merge.ts index 7e1d2f9a485..f15960f046f 100644 --- a/packages/adapter-utils/src/workspace-restore-merge.ts +++ b/packages/adapter-utils/src/workspace-restore-merge.ts @@ -556,6 +556,16 @@ export async function withDirectoryPublicationLock( }, env); } +export async function tryWithDirectoryMergeLock(targetDir: string, fn: () => Promise): Promise { + const canonicalTargetDir = await fs.realpath(targetDir); + try { + return await withCanonicalDirectoryMergeLock(canonicalTargetDir, fn, process.env, undefined, 0); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === WORKSPACE_RESTORE_LOCK_TIMEOUT_CODE) return undefined; + throw error; + } +} + async function withCanonicalDirectoryMergeLock( canonicalTargetDir: string, fn: (canonicalTargetDir: string) => Promise, diff --git a/packages/adapter-utils/src/workspace-seed-cache.test.ts b/packages/adapter-utils/src/workspace-seed-cache.test.ts index 956fbdedf5e..4dc7f99e345 100644 --- a/packages/adapter-utils/src/workspace-seed-cache.test.ts +++ b/packages/adapter-utils/src/workspace-seed-cache.test.ts @@ -5,6 +5,12 @@ import path from "node:path"; import { afterEach, expect, it, vi } from "vitest"; import { publishWorkspaceSeedGeneration, readWorkspaceSeedGeneration } from "./workspace-seed-cache.js"; +function deferred() { + let resolve!: (value: T | PromiseLike) => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} + const roots: string[] = []; afterEach(async () => { vi.restoreAllMocks(); vi.unstubAllEnvs(); @@ -112,3 +118,48 @@ it("treats optional verification storage failure as a cache miss", async () => { expect(await fs.readdir(f.company)).toEqual([]); expect(await fs.readFile(f.archive, "utf8")).toBe("run recovery bytes"); }); + +it("does not hold the company publication lock during slow verification", async () => { + const f = await fixture(); + const entered = deferred(); + const release = deferred(); + const slow = publishWorkspaceSeedGeneration(f.workspace, generation(1), f.seed, { + companyDirectory: f.company, verify: () => { entered.resolve(); return release.promise; }, + }); + try { + await entered.promise; + expect(await publishWorkspaceSeedGeneration(path.join(f.company, "second"), generation(2), f.seed, + { companyDirectory: f.company })).toBe(true); + expect(await fs.readdir(f.company)).toEqual(["second"]); + } finally { release.resolve(true); await slow; } + expect(await readWorkspaceSeedGeneration(f.workspace, generation(1))).not.toBeNull(); +}); + +it("rechecks aggregate capacity after concurrent verifications finish", async () => { + const f = await fixture(); + const release = deferred(); + const entered = vi.fn(() => release.promise); + const publications = ["first", "second"].map((name, index) => publishWorkspaceSeedGeneration( + path.join(f.company, name), generation(index), f.seed, + { companyDirectory: f.company, limits: { companyGenerations: 1 }, verify: entered }, + )); + try { await vi.waitFor(() => expect(entered).toHaveBeenCalledTimes(2)); } + finally { release.resolve(true); } + expect((await Promise.all(publications)).sort()).toEqual([false, true]); + expect(await fs.readdir(f.company)).toHaveLength(1); + expect(await fs.readFile(f.archive, "utf8")).toBe("run recovery bytes"); +}); + +it("rejects a company directory replaced during verification", async () => { + const f = await fixture(); + const replacement = path.join(f.root, "other-company"); await fs.mkdir(replacement); + expect(await publishWorkspaceSeedGeneration(f.workspace, generation(1), f.seed, { + companyDirectory: f.company, verify: async () => { + await fs.rename(f.company, `${f.company}-retained`); + await fs.symlink(replacement, f.company); + return true; + }, + })).toBe(false); + expect(await fs.readdir(replacement)).toEqual([]); + expect(await fs.readFile(f.archive, "utf8")).toBe("run recovery bytes"); +}); diff --git a/packages/adapter-utils/src/workspace-seed-cache.ts b/packages/adapter-utils/src/workspace-seed-cache.ts index c35bcd4dfcc..c26c23f7626 100644 --- a/packages/adapter-utils/src/workspace-seed-cache.ts +++ b/packages/adapter-utils/src/workspace-seed-cache.ts @@ -83,7 +83,10 @@ export async function publishWorkspaceSeedGeneration( await fs.mkdir(company, { recursive: true, mode: 0o700 }); const companyStat = await fs.lstat(company); if (!companyStat.isDirectory() || companyStat.isSymbolicLink()) return false; - return await withDirectoryMergeLock(company, async () => { + const capacity = async (): Promise<"existing" | "available" | false> => { + const currentCompany = await fs.lstat(company); + if (!currentCompany.isDirectory() || currentCompany.isSymbolicLink() + || currentCompany.dev !== companyStat.dev || currentCompany.ino !== companyStat.ino) return false; let totalGenerations = 0, totalBytes = 0, workspaces = 0; let workspace = { generations: 0, bytes: 0 }; let workspaceExists = false; @@ -99,7 +102,7 @@ export async function publishWorkspaceSeedGeneration( if (totalGenerations > limits.companyGenerations || totalBytes > limits.companyBytes) return false; if (directory === root) { workspace = usage; workspaceExists = true; } } - if (workspaceExists && await fs.lstat(path.join(root, generation)).then(() => true, () => false)) return true; + if (workspaceExists && await fs.lstat(path.join(root, generation)).then(() => true, () => false)) return "existing"; let incomingBytes = 4096; for (const archive of [seed.workspaceArchivePath, seed.gitArchivePath]) { if (!archive) continue; @@ -111,7 +114,14 @@ export async function publishWorkspaceSeedGeneration( || totalGenerations + 1 > limits.companyGenerations || totalBytes + incomingBytes > limits.companyBytes || workspaces + (workspaceExists ? 0 : 1) > limits.companyWorkspaces) return false; assertWorkspaceManifestDiskSpace(company, incomingBytes); - if (options.verify && !await options.verify()) return false; + return "available"; + }; + const initialCapacity = await capacity(); + if (initialCapacity !== "available") return initialCapacity === "existing"; + if (options.verify && !await options.verify()) return false; + return await withDirectoryMergeLock(company, async () => { + const currentCapacity = await capacity(); + if (currentCapacity !== "available") return currentCapacity === "existing"; await fs.mkdir(root, { recursive: true, mode: 0o700 }); const temporary = path.join(root, `.pending-${randomUUID()}`); await fs.mkdir(temporary, { mode: 0o700 }); diff --git a/scripts/benchmark-repository-object-cache.ts b/scripts/benchmark-repository-object-cache.ts new file mode 100644 index 00000000000..261b9dc559c --- /dev/null +++ b/scripts/benchmark-repository-object-cache.ts @@ -0,0 +1,62 @@ +import assert from "node:assert/strict"; +import { execFile as execute } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { materializeManagedProjectWorkspace } from "../server/src/services/managed-repository-checkout.js"; + +const execFile = promisify(execute); +const root = await fs.mkdtemp(path.join(os.tmpdir(), "repository-cache-benchmark-")); +const previousHome = process.env.PAPERCLIP_HOME; +const previousTrace = process.env.GIT_TRACE_PACKFILE; +process.env.PAPERCLIP_HOME = path.join(root, "home"); +const source = path.join(root, "source"); +const cache = path.join(root, "cache/company"); +const samples = Number(process.argv.find(arg => arg.startsWith("--samples="))?.split("=")[1] ?? 5); +assert.ok(Number.isSafeInteger(samples) && samples > 0 && samples <= 10); +try { + await fs.mkdir(source); + const git = async (...args: string[]) => (await execFile("git", ["-C", source, ...args])).stdout; + await git("init", "-b", "main"); + await git("config", "user.name", "Fixture"); await git("config", "user.email", "fixture@example.test"); + for (let batch = 0; batch < 4; batch++) { + await Promise.all(Array.from({ length: 32 }, (_, index) => fs.writeFile(path.join(source, `file-${batch * 32 + index}.bin`), randomBytes(256 * 1024)))); + await git("add", "."); await git("commit", "-m", `History ${batch}`); + } + const repoUrl = pathToFileURL(source).href; + const prepare = (target: string, useCache: boolean) => materializeManagedProjectWorkspace(target, + { repoUrl, ...(useCache ? { objectCacheCompanyDirectory: cache } : {}) }); + const warmStart = performance.now(); await prepare(path.join(root, "prime"), true); + const cachePrimeMs = performance.now() - warmStart; + const expected = await git("rev-parse", "HEAD"); + const results = []; + for (let sample = 0; sample < samples; sample++) { + for (const mode of sample % 2 ? ["after", "before"] : ["before", "after"]) { + const target = path.join(root, `${mode}-${sample}`); + const trace = path.join(root, `${mode}-${sample}.pack`); + process.env.GIT_TRACE_PACKFILE = trace; + const start = performance.now(); await prepare(target, mode === "after"); + const wallMs = performance.now() - start; + const receivedPackBytes = await fs.stat(trace).then(stat => stat.size, () => 0); + delete process.env.GIT_TRACE_PACKFILE; + assert.equal((await execFile("git", ["-C", target, "rev-parse", "HEAD"])).stdout, expected); + await assert.rejects(fs.stat(path.join(target, ".git/objects/info/alternates")), { code: "ENOENT" }); + await execFile("git", ["-C", target, "fsck", "--full"]); + results.push({ mode, sample, wallMs, receivedPackBytes }); + await fs.rm(target, { recursive: true, force: true }); + } + } + const summary = Object.fromEntries(["before", "after"].map(mode => { + const timings = results.filter(result => result.mode === mode).map(result => result.wallMs).sort((a, b) => a - b); + return [mode, { medianMs: timings[Math.floor(timings.length / 2)], minMs: timings[0], maxMs: timings.at(-1) }]; + })); + console.log(JSON.stringify({ node: process.version, platform: process.platform, transport: "local file:// Git upload-pack; no injected latency", + files: 128, contentBytes: 32 * 1024 * 1024, commits: 4, samples, cachePrimeMs, summary, results }, null, 2)); +} finally { + if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome; + if (previousTrace === undefined) delete process.env.GIT_TRACE_PACKFILE; else process.env.GIT_TRACE_PACKFILE = previousTrace; + await fs.rm(root, { recursive: true, force: true }); +} diff --git a/scripts/benchmark-workspace-staging.ts b/scripts/benchmark-workspace-staging.ts new file mode 100644 index 00000000000..556fde84446 --- /dev/null +++ b/scripts/benchmark-workspace-staging.ts @@ -0,0 +1,116 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { setTimeout as delay } from "node:timers/promises"; +import { prepareSandboxManagedRuntime, type SandboxSyncOperation } from "../packages/adapter-utils/src/sandbox-managed-runtime.js"; +import { captureDirectorySnapshot, directorySnapshotSha256, disposeDirectorySnapshot } from "../packages/adapter-utils/src/workspace-restore-merge.js"; + +const execute = promisify(execFile); +const argument = (name: string, fallback: string) => process.argv.find(arg => arg.startsWith(`--${name}=`))?.slice(name.length + 3) ?? fallback; +const files = Number(argument("files", "10000")); +const samples = Number(argument("samples", "5")); +const transferDelayMs = Number(argument("transfer-delay-ms", "0")); +const baselineRef = argument("baseline-ref", "origin/master"); +assert.ok(Number.isSafeInteger(files) && files > 0 && files <= 100000); +assert.ok(Number.isSafeInteger(samples) && samples > 0 && samples <= 10); +assert.ok(Number.isSafeInteger(transferDelayMs) && transferDelayMs >= 0 && transferDelayMs <= 10000); +const repository = path.resolve(import.meta.dirname, ".."); +const root = await fs.mkdtemp(path.join(os.tmpdir(), "workspace-staging-benchmark-")); +const previousHome = process.env.PAPERCLIP_HOME; +process.env.PAPERCLIP_HOME = path.join(root, "home"); +const moduleDirectory = path.join(repository, "packages/adapter-utils/src"); +const token = randomUUID(); +const baselineModule = path.join(moduleDirectory, `sandbox-managed-runtime.benchmark-${token}.ts`); +const baselineCache = path.join(moduleDirectory, `workspace-seed-cache.benchmark-${token}.ts`); +try { + const revision = (await execute("git", ["rev-parse", baselineRef], { cwd: repository })).stdout.trim(); + const source = async (file: string) => (await execute("git", ["show", `${revision}:packages/adapter-utils/src/${file}.ts`], + { cwd: repository, maxBuffer: 1024 * 1024 })).stdout; + await fs.writeFile(baselineCache, await source("workspace-seed-cache")); + await fs.writeFile(baselineModule, (await source("sandbox-managed-runtime")) + .replace('"./workspace-seed-cache.js"', `"./${path.basename(baselineCache)}"`)); + const baseline = (await import(pathToFileURL(baselineModule).href)).prepareSandboxManagedRuntime as typeof prepareSandboxManagedRuntime; + const host = path.join(root, "host"); await fs.mkdir(host); + const git = (...args: string[]) => execute("git", ["-C", host, ...args]); + await git("init", "-b", "main"); + for (let start = 0; start < files; start += 32) { + await Promise.all(Array.from({ length: Math.min(32, files - start) }, async (_, offset) => { + const index = start + offset; + const dir = path.join(host, `directory-${Math.floor(index / 100)}`); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, `file-${index}.txt`), `${index}\n${"Synthetic workspace data\n".repeat(160)}`); + })); + } + await fs.writeFile(path.join(host, ".gitignore"), "ignored.txt\n"); + await git("add", "."); + await git("-c", "user.name=Fixture", "-c", "user.email=fixture@example.test", "commit", "-m", "fixture"); + await fs.writeFile(path.join(host, "directory-0", "file-0.txt"), "uncommitted bytes\n"); + await fs.unlink(path.join(host, "directory-0", "file-1.txt")); + await fs.writeFile(path.join(host, "untracked.txt"), "new bytes\n"); + await fs.writeFile(path.join(host, "ignored.txt"), "must stay private\n"); + const results = []; + for (let sample = 0; sample < samples; sample++) { + for (const mode of sample % 2 ? ["after", "before"] : ["before", "after"]) { + const prepare = mode === "before" ? baseline : prepareSandboxManagedRuntime; + const location = path.join(root, `${mode}-${sample}`); await fs.mkdir(location); + const remote = path.join(location, "remote"); + let bytes = 0; + const phases: Record = {}; + const run = async (command: string) => { await execute("sh", ["-c", command]); }; + const sync = async (operations: SandboxSyncOperation[]) => { + if (transferDelayMs) await delay(transferDelayMs); + for (const operation of operations) { + for (const file of operation.files) { + bytes += (await fs.stat(file.sourcePath)).size; + await fs.mkdir(path.dirname(file.targetPath), { recursive: true }); + await fs.copyFile(file.sourcePath, file.targetPath); + } + for (const command of operation.postUploadCommands ?? []) await run(command.command); + } + return { operations: [] }; + }; + const start = performance.now(); + const runtime = await prepare({ + spec: { provider: "test", sandboxId: "test", remoteCwd: remote, timeoutMs: 30000, apiKey: null, transport: "sandbox" }, + adapterKey: "benchmark", workspaceLocalDir: host, + workspaceSeedCacheDirectory: path.join(location, "cache"), + workspaceDurableSeed: { workspaceArchivePath: path.join(location, "workspace.tar"), gitArchivePath: path.join(location, "git.tar") }, + runtimeSpan: async (name, work) => { + const phaseStart = performance.now(); + try { return await work(); } + finally { phases[name] = (phases[name] ?? 0) + performance.now() - phaseStart; } + }, + client: { + makeDir: dir => fs.mkdir(dir, { recursive: true }).then(() => {}), run, + writeFile: (file, data) => fs.writeFile(file, Buffer.from(data)), readFile: file => fs.readFile(file), + listFiles: dir => fs.readdir(dir), remove: file => fs.rm(file, { force: true, recursive: true }), syncIn: sync, + }, + }); + const wallMs = performance.now() - start; + const snapshot = runtime.workspaceSyncSnapshot!; + const actual = await captureDirectorySnapshot(remote, { exclude: snapshot.baseline.exclude, ignoredPaths: snapshot.baseline.ignoredPaths, diskBacked: true }); + try { assert.equal(directorySnapshotSha256(actual), directorySnapshotSha256(snapshot.baseline)); } + finally { await disposeDirectorySnapshot(actual); await runtime.cleanupWorkspaceSnapshot(); } + assert.equal((await execute("git", ["-C", remote, "rev-parse", "HEAD"])).stdout, (await git("rev-parse", "HEAD")).stdout); + await assert.rejects(fs.stat(path.join(remote, "ignored.txt")), { code: "ENOENT" }); + assert.equal((await fs.readdir(path.join(location, "cache"))).length, 1); + results.push({ mode, sample, wallMs, bytes, phases }); + await fs.rm(location, { recursive: true, force: true }); + } + } + const summary = Object.fromEntries(["before", "after"].map(mode => { + const timings = results.filter(result => result.mode === mode).map(result => result.wallMs).sort((a, b) => a - b); + return [mode, { medianMs: timings[Math.floor(timings.length / 2)], minMs: timings[0], maxMs: timings.at(-1) }]; + })); + console.log(JSON.stringify({ node: process.version, platform: process.platform, baselineRef: revision, + files, samples, transferDelayMs, transport: "local copy and shell extraction; optional controlled transfer delay", summary, results }, null, 2)); +} finally { + if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; + else process.env.PAPERCLIP_HOME = previousHome; + await Promise.all([root, baselineModule, baselineCache].map(file => fs.rm(file, { force: true, recursive: true }))); +} diff --git a/server/src/__tests__/repository-object-cache.test.ts b/server/src/__tests__/repository-object-cache.test.ts new file mode 100644 index 00000000000..4c5454d6f81 --- /dev/null +++ b/server/src/__tests__/repository-object-cache.test.ts @@ -0,0 +1,183 @@ +import { execFile as execute } from "node:child_process"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { createServer } from "node:http"; +import { createReadStream } from "node:fs"; +import { afterEach, expect, it, vi } from "vitest"; +import { withDirectoryMergeLock } from "@paperclipai/adapter-utils/workspace-restore-merge"; +import { materializeManagedProjectWorkspace } from "../services/managed-repository-checkout.js"; +import { assertReproducibleBaseGitStorage } from "../services/git-workspace-restoration.js"; +import { readRepositoryObjectCache, resolveRepositoryObjectCache, publishRepositoryObjectCache } from "../services/repository-object-cache.js"; + +const execFile = promisify(execute); +const roots: string[] = []; +afterEach(async () => { + vi.unstubAllEnvs(); + await Promise.all(roots.splice(0).map(root => fs.rm(root, { recursive: true, force: true }))); +}); +async function fixture() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "repository-object-cache-")); roots.push(root); + vi.stubEnv("PAPERCLIP_HOME", path.join(root, "home")); + const source = path.join(root, "source"); await fs.mkdir(source); + const git = async (cwd: string, ...args: string[]) => (await execFile("git", ["-C", cwd, ...args])).stdout.trim(); + await git(source, "init", "-b", "main"); + await git(source, "config", "user.name", "Fixture"); await git(source, "config", "user.email", "fixture@example.test"); + await fs.writeFile(path.join(source, "work.txt"), "initial\n"); + await git(source, "add", "."); await git(source, "commit", "-m", "initial"); + const repoUrl = pathToFileURL(source).href; + const company = path.join(root, "cache", "company"); + const prepare = (name: string) => materializeManagedProjectWorkspace(path.join(root, name), { repoUrl, objectCacheCompanyDirectory: company }); + return { root, source, git, company, repoUrl, prepare }; +} + +it("uses current remote refs and produces an independent checkout without private cached refs", async () => { + const f = await fixture(); + await f.git(f.source, "checkout", "-b", "private"); + await fs.writeFile(path.join(f.source, "private.txt"), "private branch bytes"); + await f.git(f.source, "add", "."); await f.git(f.source, "commit", "-m", "private"); + const privateHead = await f.git(f.source, "rev-parse", "HEAD"); + await f.git(f.source, "checkout", "main"); + await f.prepare("first"); + const cache = (await resolveRepositoryObjectCache(f.company, f.repoUrl, null))!; + expect(await readRepositoryObjectCache(cache)).toBe(cache.directory); + await f.git(f.source, "branch", "-D", "private"); + await fs.writeFile(path.join(f.source, "work.txt"), "current\n"); + await f.git(f.source, "commit", "-am", "current"); await f.git(f.source, "tag", "current-tag"); + const currentHead = await f.git(f.source, "rev-parse", "HEAD"); + await f.prepare("second"); + const target = path.join(f.root, "second"); + expect(await f.git(target, "rev-parse", "HEAD")).toBe(currentHead); + expect(await f.git(target, "rev-parse", "current-tag")).toBe(currentHead); + expect(await fs.readFile(path.join(target, "work.txt"), "utf8")).toBe("current\n"); + await expect(f.git(target, "cat-file", "-e", privateHead)).rejects.toThrow(); + await expect(fs.stat(path.join(target, ".git/objects/info/alternates"))).rejects.toMatchObject({ code: "ENOENT" }); + await assertReproducibleBaseGitStorage(target); + await fs.rm(f.company, { recursive: true }); + await f.git(target, "fsck", "--full"); + expect(await f.git(target, "show", "HEAD~1:work.txt")).toBe("initial"); +}); + +it("never substitutes a cached checkout for an unavailable remote", async () => { + const f = await fixture(); await f.prepare("first"); + await fs.rename(f.source, `${f.source}-unavailable`); + await expect(f.prepare("denied")).rejects.toThrow("Failed to prepare managed checkout"); + await expect(fs.stat(path.join(f.root, "denied"))).rejects.toMatchObject({ code: "ENOENT" }); +}); + +it("separates company and current credential scopes", async () => { + const f = await fixture(); + const auth = { source: "company_secret" as const, secretName: "GITHUB_TOKEN", configArgs: [], env: { TOKEN: "first-token" } }; + const first = await resolveRepositoryObjectCache(f.company, f.repoUrl, auth); + const other = await resolveRepositoryObjectCache(f.company, f.repoUrl, { ...auth, env: { TOKEN: "new-token" } }); + const company = await resolveRepositoryObjectCache(path.join(f.root, "cache/other-company"), f.repoUrl, auth); + expect(first!.directory).not.toBe(other!.directory); expect(first!.directory).not.toBe(company!.directory); + expect(JSON.stringify(first)).not.toContain("first-token"); +}); + +it("skips contended optional cache publication without delaying a fresh checkout", async () => { + const f = await fixture(); + await resolveRepositoryObjectCache(f.company, f.repoUrl, null); + await withDirectoryMergeLock(f.company, async () => { + await f.prepare("first"); + expect(await fs.readdir(f.company)).toEqual([]); + }); + expect(await fs.readFile(path.join(f.root, "first/work.txt"), "utf8")).toBe("initial\n"); +}); + +it("rebuilds from the remote when the cached pack is corrupt", async () => { + const f = await fixture(); await f.prepare("first"); + const cache = (await resolveRepositoryObjectCache(f.company, f.repoUrl, null))!; + await f.git(cache.directory, "gc", "--prune=now"); + const packs = (await fs.readdir(path.join(cache.directory, "objects/pack"))).filter(entry => entry.endsWith(".pack")); + expect(packs.length).toBeGreaterThan(0); + for (const entry of packs) { + const file = path.join(cache.directory, "objects/pack", entry); + await fs.chmod(file, 0o600); + await fs.writeFile(file, "corrupted"); + } + await f.git(path.join(f.root, "first"), "fsck", "--full"); + await f.prepare("second"); + expect(await fs.readFile(path.join(f.root, "second/work.txt"), "utf8")).toBe("initial\n"); +}); + +it("still authenticates to the HTTP remote when a matching object cache exists", async () => { + const f = await fixture(); + const remote = path.join(f.root, "remote.git"); + await execFile("git", ["clone", "--bare", f.source, remote]); + await f.git(remote, "update-server-info"); + let revoked = false, rejected = 0; + const server = createServer(async (request, response) => { + if (revoked || request.headers.authorization !== "Bearer fixture-token") { + rejected++; response.writeHead(403); response.end(); return; + } + const pathname = new URL(request.url!, "http://localhost").pathname; + if (!/^\/[a-zA-Z0-9/._-]+$/.test(pathname) || pathname.includes("..")) { response.writeHead(404); response.end(); return; } + const file = path.join(remote, pathname); + if (!await fs.stat(file).then(stat => stat.isFile(), () => false)) { response.writeHead(404); response.end(); return; } + response.setHeader("Content-Type", "text/plain"); + createReadStream(file).pipe(response); + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + try { + const address = server.address() as { port: number }; + const repoUrl = `http://127.0.0.1:${address.port}`; + const resolveGitAuth = vi.fn(async () => ({ source: "company_secret" as const, secretName: "fixture", configArgs: [], env: { + GIT_CONFIG_COUNT: "1", GIT_CONFIG_KEY_0: "http.extraHeader", GIT_CONFIG_VALUE_0: "Authorization: Bearer fixture-token", + } })); + const prepare = (name: string) => materializeManagedProjectWorkspace(path.join(f.root, name), { + repoUrl, objectCacheCompanyDirectory: f.company, resolveGitAuth, + }); + await prepare("authorized"); + expect(await fs.readdir(f.company)).toHaveLength(1); + revoked = true; + await expect(prepare("revoked")).rejects.toThrow("403"); + expect(rejected).toBeGreaterThan(0); expect(resolveGitAuth.mock.calls.length).toBeGreaterThanOrEqual(2); + await expect(fs.stat(path.join(f.root, "revoked"))).rejects.toMatchObject({ code: "ENOENT" }); + expect(await fs.readFile(path.join(f.root, "authorized/.git/config"), "utf8")).not.toContain("fixture-token"); + } finally { await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); } +}); + +it("does not follow a cache directory symlink or grow an over-capacity company", async () => { + const f = await fixture(); + const cache = (await resolveRepositoryObjectCache(f.company, f.repoUrl, null))!; + await fs.symlink(f.source, cache.directory); + expect(await readRepositoryObjectCache(cache)).toBeNull(); + await fs.unlink(cache.directory); + for (let i = 0; i < 64; i++) await fs.mkdir(path.join(f.company, `retained-${i}`)); + await publishRepositoryObjectCache(cache, f.source); + expect(await fs.readdir(f.company)).toHaveLength(64); + await expect(fs.stat(cache.directory)).rejects.toMatchObject({ code: "ENOENT" }); +}); + +it("skips oversized repositories without creating a cache generation", async () => { + const f = await fixture(); + const cache = (await resolveRepositoryObjectCache(f.company, f.repoUrl, null))!; + const oversized = path.join(f.source, ".git/oversized"); + await fs.writeFile(oversized, ""); await fs.truncate(oversized, 2 * 1024 ** 3 + 1); + await publishRepositoryObjectCache(cache, f.source); + expect(await fs.readdir(f.company)).toEqual([]); + expect(await f.git(f.source, "show", "HEAD:work.txt")).toBe("initial"); +}); + +it.each(["https://user:secret@github.com/fixture/repo", "https://github.com/fixture/repo?token=secret", "https://github.com/fixture/repo#secret"])("does not persist credential-bearing URLs: %s", async (repoUrl) => { + const f = await fixture(); + expect(await resolveRepositoryObjectCache(f.company, repoUrl, null)).toBeNull(); + await expect(fs.stat(f.company)).rejects.toMatchObject({ code: "ENOENT" }); +}); + +it("does not initialize or publish through a replaced cache parent", async () => { + const f = await fixture(); + const other = path.join(f.root, "other"); await fs.mkdir(other); + await fs.symlink(other, path.dirname(f.company)); + expect(await resolveRepositoryObjectCache(f.company, f.repoUrl, null)).toBeNull(); + expect(await fs.readdir(other)).toEqual([]); + await fs.unlink(path.dirname(f.company)); + const cache = (await resolveRepositoryObjectCache(f.company, f.repoUrl, null))!; + await fs.rename(path.dirname(f.company), path.join(f.root, "retained")); + await fs.mkdir(path.join(other, "company")); await fs.symlink(other, path.dirname(f.company)); + await publishRepositoryObjectCache(cache, f.source); + expect(await fs.readdir(path.join(other, "company"))).toEqual([]); +}); diff --git a/server/src/services/execution-preparation-timing.ts b/server/src/services/execution-preparation-timing.ts index baa3858b33a..94f1a67862b 100644 --- a/server/src/services/execution-preparation-timing.ts +++ b/server/src/services/execution-preparation-timing.ts @@ -43,6 +43,7 @@ export type ExecutionPhase = | "workspace.materialize.directories" | "workspace.materialize.source_identity" | "workspace.checkout.lookup" | "workspace.checkout.git_auth" | "workspace.checkout.git_clone" | "workspace.checkout.git_ref" | "workspace.checkout.publish" + | "workspace.checkout.cache_lookup" | "workspace.checkout.cache_publish" | "runtime_selection.review" | "runtime_selection.contract_read" | "runtime_selection.contract_ensure" | "runtime_selection.resume" | "runtime_selection.recovery_profile" | "runtime_selection.profile" | "runtime_selection.interactions" | "runtime_selection.provider_discovery" | "runtime_selection.bootstrap" | "runtime_selection.run_write" | "runtime_selection.finalization_write" diff --git a/server/src/services/isolated-remote-workspace-source.ts b/server/src/services/isolated-remote-workspace-source.ts index c0bf8250db1..ac5f7ee5be6 100644 --- a/server/src/services/isolated-remote-workspace-source.ts +++ b/server/src/services/isolated-remote-workspace-source.ts @@ -6,7 +6,7 @@ import fs from "node:fs/promises"; import { execFile as execFileCallback } from "node:child_process"; import { promisify } from "node:util"; import type { TaskWorkspaceIntent } from "@paperclipai/shared"; -import type { GitRemoteAuthProvider } from "./git-credentials.js"; +import { isGitHubHttpsRemoteUrl, type GitRemoteAuthProvider } from "./git-credentials.js"; import { resolvePaperclipInstanceRoot } from "../home-paths.js"; import { materializeManagedProjectWorkspace } from "./managed-repository-checkout.js"; import { measureRunPhase } from "./execution-preparation-timing.js"; @@ -139,6 +139,8 @@ export async function materializeIsolatedRemoteWorkspaceSource(input: { }); const result = await materializeManagedProjectWorkspace(canonicalCwd, { repoUrl: input.repoUrl, resolveGitAuth: input.resolveGitAuth, + objectCacheCompanyDirectory: isGitHubHttpsRemoteUrl(input.repoUrl) + ? path.join(await fs.realpath(resolvePaperclipInstanceRoot()), "repository-object-cache", input.companyId) : undefined, }); if (result.warning) throw new Error("Isolated repository source is not a Git checkout"); await measureRunPhase("workspace.materialize.source_identity", () => assertRemoteSourceIdentity(cwd, input.repoUrl)); diff --git a/server/src/services/managed-repository-checkout.ts b/server/src/services/managed-repository-checkout.ts index e8316869473..1e6205381b4 100644 --- a/server/src/services/managed-repository-checkout.ts +++ b/server/src/services/managed-repository-checkout.ts @@ -9,6 +9,7 @@ import { describeGitAuthFailure, scrubGitCredentialText, type GitRemoteAuthProvi import { classifyGitCloneFailure, GitConnectionFailureError, readGitConnectionFailure } from "./git-connection-failure.js"; import { isWorkspaceGitScanError, WorkspaceGitScanError } from "./workspace-git-operation-scheduler.js"; import { measureRunPhase } from "./execution-preparation-timing.js"; +import { resolveRepositoryObjectCache, readRepositoryObjectCache, publishRepositoryObjectCache } from "./repository-object-cache.js"; const execFile = promisify(execFileCallback); const MANAGED_WORKSPACE_GIT_CLONE_TIMEOUT_MS = 10 * 60 * 1000; @@ -23,6 +24,7 @@ export async function materializeManagedProjectWorkspace( beforeCopy?: (cloneCwd: string) => Promise; beforePublish?: (cloneCwd: string) => Promise; stagingParent?: string; + objectCacheCompanyDirectory?: string; }, ): Promise<{ cwd: string; warning: string | null }> { await fs.mkdir(path.dirname(cwd), { recursive: true }); @@ -55,19 +57,22 @@ export async function materializeManagedProjectWorkspace( await fs.rm(cwd, { recursive: true, force: true }); } - const auth = input.resolveGitAuth && !input.localSource + let auth = input.resolveGitAuth && !input.localSource ? await measureRunPhase("workspace.checkout.git_auth", () => input.resolveGitAuth!(input.repoUrl!)) : null; const cloneTmpDir = await fs.mkdtemp(input.stagingParent ? path.join(input.stagingParent, `${path.basename(cwd)}.clone-`) : `${cwd}.clone-`); const cloneSource = input.localSource ?? input.repoUrl; + let objectCache = input.objectCacheCompanyDirectory && !input.localSource + ? await measureRunPhase("workspace.checkout.cache_lookup", () => resolveRepositoryObjectCache(input.objectCacheCompanyDirectory!, input.repoUrl!, auth)) : null; + const reference = objectCache ? await measureRunPhase("workspace.checkout.cache_lookup", () => readRepositoryObjectCache(objectCache!)) : null; try { await input.beforeCopy?.(cloneTmpDir); try { - await measureRunPhase("workspace.checkout.git_clone", () => execFile( + const clone = (referencePath: string | null) => measureRunPhase("workspace.checkout.git_clone", () => execFile( "git", - [...(auth?.configArgs ?? []), "clone", "--no-hardlinks", "--", cloneSource, cloneTmpDir], + [...(auth?.configArgs ?? []), ...(referencePath ? ["-c", "repack.updateServerInfo=false"] : []), "clone", "--no-hardlinks", ...(referencePath ? ["--reference-if-able", referencePath, "--dissociate"] : []), "--", cloneSource, cloneTmpDir], { env: { ...sanitizeRuntimeServiceBaseEnv(process.env), @@ -77,6 +82,14 @@ export async function materializeManagedProjectWorkspace( timeout: MANAGED_WORKSPACE_GIT_CLONE_TIMEOUT_MS, }, )); + try { await clone(reference); } + catch (error) { + if (!reference) throw error; + for (const name of await fs.readdir(cloneTmpDir)) await fs.rm(path.join(cloneTmpDir, name), { recursive: true, force: true }); + objectCache = null; + auth = input.resolveGitAuth ? await input.resolveGitAuth(input.repoUrl) : null; + await clone(null); + } } catch (error) { const connectionFailure = !input.localSource ? classifyGitCloneFailure(input.repoUrl, error) : null; if (connectionFailure) throw new GitConnectionFailureError( @@ -84,6 +97,7 @@ export async function materializeManagedProjectWorkspace( ); throw error; } + if (objectCache && !reference) await measureRunPhase("workspace.checkout.cache_publish", () => publishRepositoryObjectCache(objectCache!, cloneTmpDir)); if (input.localSource) { const snapshot = await readGitWorkspaceSnapshot(input.localSource, false); if (!snapshot) throw new Error("Configured repository folder is not a Git checkout"); diff --git a/server/src/services/repository-object-cache.ts b/server/src/services/repository-object-cache.ts new file mode 100644 index 00000000000..9a0590739f2 --- /dev/null +++ b/server/src/services/repository-object-cache.ts @@ -0,0 +1,111 @@ +import { createHash, randomUUID } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { execFile as execute } from "node:child_process"; +import { promisify } from "node:util"; +import { tryWithDirectoryMergeLock } from "@paperclipai/adapter-utils/workspace-restore-merge"; +import { assertWorkspaceManifestDiskSpace } from "@paperclipai/adapter-utils/workspace-manifest"; +import type { GitAuthInvocation } from "./git-credentials.js"; +import { sanitizeRuntimeServiceBaseEnv } from "./runtime-service-env.js"; + +const execFile = promisify(execute); +const REPOSITORY_BYTES = 2 * 1024 ** 3; +const COMPANY_BYTES = 4 * 1024 ** 3; +const COMPANY_REPOSITORIES = 64; +const hash = (value: string) => createHash("sha256").update(value).digest("hex"); + +export interface RepositoryObjectCache { + companyDirectory: string; + directory: string; + key: string; + repoUrl: string; +} + +export async function resolveRepositoryObjectCache(companyDirectory: string, repoUrl: string, auth: GitAuthInvocation | null): Promise { + try { + const remote = new URL(repoUrl); + if (remote.username || remote.password || remote.search || remote.hash) return null; + const parent = path.dirname(companyDirectory); + await fs.mkdir(parent, { recursive: true, mode: 0o700 }); + for (const directory of [parent, companyDirectory]) { + if (directory === companyDirectory) await fs.mkdir(directory, { mode: 0o700 }).catch((error: NodeJS.ErrnoException) => { + if (error.code !== "EEXIST") throw error; + }); + const stat = await fs.lstat(directory); + if (!stat.isDirectory() || stat.isSymbolicLink()) return null; + } + companyDirectory = await fs.realpath(companyDirectory); + const scope = auth ? [auth.source, auth.secretName, auth.configArgs, Object.entries(auth.env).sort(([a], [b]) => a.localeCompare(b))] : null; + const key = hash(JSON.stringify([repoUrl, scope])); + return { companyDirectory, directory: path.join(companyDirectory, key), key, repoUrl }; + } catch { return null; } +} + +export async function readRepositoryObjectCache(cache: RepositoryObjectCache): Promise { + try { + for (const directory of [cache.companyDirectory, cache.directory]) { + const stat = await fs.lstat(directory); + if (!stat.isDirectory() || stat.isSymbolicLink()) return null; + } + if (await fs.realpath(cache.companyDirectory) !== cache.companyDirectory) return null; + const objects = await fs.lstat(path.join(cache.directory, "objects")); + if (!objects.isDirectory() || objects.isSymbolicLink()) return null; + if (await fs.lstat(path.join(cache.directory, "objects/info/alternates")).then(() => true, () => false)) return null; + const receiptPath = path.join(cache.directory, "paperclip-object-cache.json"); + const stat = await fs.lstat(receiptPath); + if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 4096) return null; + const receipt = JSON.parse(await fs.readFile(receiptPath, "utf8")); + if (receipt.version !== 1 || receipt.key !== cache.key) return null; + const options = { timeout: 10000, maxBuffer: 16384, env: sanitizeRuntimeServiceBaseEnv(process.env) }; + const bare = await execFile("git", ["-C", cache.directory, "rev-parse", "--is-bare-repository"], options); + const origin = await execFile("git", ["-C", cache.directory, "config", "--local", "--null", "--get-all", "remote.origin.url"], options); + if (bare.stdout.trim() !== "true" || origin.stdout !== `${cache.repoUrl}\0`) return null; + return cache.directory; + } catch { return null; } +} + +async function storageBytes(directory: string, ceiling: number, budget: { entries: number; deadline: number }): Promise { + let bytes = 0; + for await (const entry of await fs.opendir(directory)) { + if (--budget.entries < 0 || performance.now() > budget.deadline) throw new Error("Repository object cache work budget exhausted"); + const target = path.join(directory, entry.name); + const stat = await fs.lstat(target); + if (stat.isSymbolicLink()) throw new Error("Invalid repository object cache entry"); + bytes += stat.isDirectory() ? await storageBytes(target, ceiling - bytes, budget) : stat.size; + if (bytes > ceiling) throw new Error("Repository object cache capacity exhausted"); + } + return bytes; +} + +export async function publishRepositoryObjectCache(cache: RepositoryObjectCache, checkout: string): Promise { + try { + if (await readRepositoryObjectCache(cache)) return; + await tryWithDirectoryMergeLock(cache.companyDirectory, async () => { + const company = await fs.lstat(cache.companyDirectory); + if (!company.isDirectory() || company.isSymbolicLink() || await fs.realpath(cache.companyDirectory) !== cache.companyDirectory) return; + const budget = { entries: 4096, deadline: performance.now() + 5000 }; + let repositories = 0, bytes = 0; + for await (const entry of await fs.opendir(cache.companyDirectory)) { + if (++repositories >= COMPANY_REPOSITORIES) return; + const directory = path.join(cache.companyDirectory, entry.name); + const stat = await fs.lstat(directory); + if (!stat.isDirectory() || stat.isSymbolicLink()) return; + bytes += await storageBytes(directory, COMPANY_BYTES - bytes, budget); + } + if (await fs.lstat(cache.directory).then(() => true, () => false)) return; + const incoming = await storageBytes(path.join(checkout, ".git"), REPOSITORY_BYTES, budget) + 65536; + if (bytes + incoming > COMPANY_BYTES) return; + assertWorkspaceManifestDiskSpace(cache.companyDirectory, incoming); + const temporary = path.join(cache.companyDirectory, `.pending-${randomUUID()}`); + try { + const options = () => ({ timeout: Math.max(1, Math.ceil(budget.deadline - performance.now())), maxBuffer: 16384, env: sanitizeRuntimeServiceBaseEnv(process.env) }); + await execFile("git", ["clone", "--bare", "--no-hardlinks", "--", checkout, temporary], options()); + await execFile("git", ["-C", temporary, "remote", "set-url", "origin", cache.repoUrl], options()); + await fs.writeFile(path.join(temporary, "paperclip-object-cache.json"), JSON.stringify({ version: 1, key: cache.key })); + const actual = await storageBytes(temporary, REPOSITORY_BYTES, budget); + if (bytes + actual > COMPANY_BYTES) return; + await fs.rename(temporary, cache.directory); + } finally { await fs.rm(temporary, { recursive: true, force: true }); } + }); + } catch {} +}