diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index a3b677a6d3b..527372b7f72 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1297,7 +1297,7 @@ publication receipts, binding checks, and lifecycle locks remain authoritative. ## Project Repository Checkouts -Tasks use every distinct repository attached to their project, including repository-only sources with no local folder. Paperclip creates a managed checkout when no local folder is configured. The selected repository remains at the task workspace root. Other project repositories have editable, independent Git checkouts under `.paperclip-repositories/-`. Workspace hints expose each checkout path to the agent. +Tasks use every distinct repository attached to their project, including repository-only sources with no local folder. Paperclip creates a managed checkout when no local folder is configured. The selected repository remains at the task workspace root. Other project repositories have editable Git checkouts under `.paperclip-repositories/-`. Workspace hints expose each checkout path to the agent. A repository without a configured local folder is checked out as a Git worktree of one shared managed clone per project and repository URL, on its own `paperclip//` branch tracking `origin/`, so its objects are stored once; see [Recoverable disk cleanup](recoverable-disk-cleanup.md#nested-project-repository-worktrees). Existing standalone checkouts keep working. When an additional repository has a configured local checkout, Paperclip seeds the task copy from its current commit and uncommitted files. Git-ignored files stay out of that copy. Subsequent task edits stay in the task copy. They do not overwrite the configured source folder. Existing task copies retain their work across runs. @@ -3012,7 +3012,10 @@ the project's repositories and it passes the same checks as the worktree state), stock Git storage (no custom hooks), no linked worktrees, no unreachable objects, and every commit reachable from its refs or reflogs is reachable from a current upstream branch or tag, read fresh from the project -repository's URL with the company's Git identity. Task repositories +repository's URL with the company's Git identity. A checkout that is a worktree of the project's shared clone is +checked against its own `HEAD`, `HEAD` reflog, index and branches under its +`paperclip//` prefix instead of the clone's storage and other tasks' refs. +Task repositories (`task-repo-*`) always prevent deletion. Inspection runs Git with `core.fsmonitor` disabled, so a repository's local config cannot make it run a command. Standard linked-worktree `FETCH_HEAD` records are retained unless every diff --git a/doc/recoverable-disk-cleanup.md b/doc/recoverable-disk-cleanup.md index 6a513c9b7ac..e7ef3ba52f4 100644 --- a/doc/recoverable-disk-cleanup.md +++ b/doc/recoverable-disk-cleanup.md @@ -214,6 +214,51 @@ still keep a worktree from expiring. Restore points are refused when the file storage is `s3` and the secrets master key does not come from `PAPERCLIP_SECRETS_MASTER_KEY`. +## Nested project repository worktrees + +Extra project repositories that have no configured local folder are no longer full +clones inside every task. Each task's `.paperclip-repositories/-` checkout is a +Git worktree of one shared clone per company, project and repository URL: the managed +project checkout for that URL under the instance's `projects/` directory (the same clone +that main task worktrees use when the repository is a task's primary source). Objects are +stored once in that clone, so a new task adds only its working files. Repositories with a +configured local folder are still seeded as independent copies, and existing standalone +nested clones keep working unchanged; they are not migrated. + +Each nested worktree checks out its own branch, `paperclip//`, where `` +derives from the worktree path and `` is the remote's current default branch +(refreshed before each new worktree) or the configured branch ref, tracking +`origin/`. Reopening derives the upstream from the saved source ref, not from the +branch name. A tag or commit ref is checked out detached, as +before. Branch names therefore never collide between tasks. When a checkout kept under +`.paperclip-runtime/detached-repositories/` still holds that branch, the new worktree +uses `paperclip//r/` instead, so the retained work is never reset. `git pull` works as before; +pushing needs an explicit destination such as `git push origin HEAD:`. Adding the +worktree, fetching and removing registrations are serialized per shared clone. Each new +worktree first fetches with `--prune`, so a branch deleted upstream is rejected as a fresh +clone would reject it. Server-side Git commands on the shared clone run with hooks +disabled, so hooks an agent plants there never run in the server. + +Branches and reflogs of a worktree live partly in the shared clone. Expiration and restore +points therefore scope their checks to what belongs to the nested worktree: its `HEAD`, +`HEAD` reflog, `ORIG_HEAD`, index and working tree (stored per worktree) and every branch +under its own `paperclip//` prefix together with that branch's reflog. Those must be on +the remote or saved in a restore point exactly as for a standalone checkout. Checking out +any other branch, stashes, unsupported refs or custom local config in the shared clone +still block expiration. Branches outside the prefix, other tasks' branches and their +history stay in the shared clone and never block or get deleted. Commits that existed only +in the nested worktree's `HEAD` reflog are written back into the restored worktree's `HEAD` +reflog, so they stay protected after reopening. + +Expiring a task removes the nested worktree's registration from the shared clone and +deletes its own branch only at the verified recorded commit. Reopening recreates the +worktree from the shared clone (re-cloning the shared clone if it is gone) on the same +branch and commit and applies its restore point. Deleting or archiving a task removes the +registration of each of its nested worktrees and deletes their branches only when every +commit is on a remote-tracking ref; an unpushed branch stays in the shared clone. Moving a +nested worktree to `.paperclip-runtime/detached-repositories/` when its repository leaves +the project repairs its registration. + Reopening the issue, or the next run that recreates the worktree, imports the bundle, re-adds the worktree at the saved commit, applies the restore point and verifies the result. The bundle is always imported, so history-only commits are present again as diff --git a/server/src/__tests__/git-workspace-expiration.test.ts b/server/src/__tests__/git-workspace-expiration.test.ts index b023805c47a..7055ede59d1 100644 --- a/server/src/__tests__/git-workspace-expiration.test.ts +++ b/server/src/__tests__/git-workspace-expiration.test.ts @@ -20,9 +20,10 @@ import { realizeExecutionWorkspace, ensurePersistedExecutionWorkspaceAvailable } import { executionWorkspaceService } from "../services/execution-workspaces.js"; import { prepareProjectRepositoryWorkspaces, mergeExecutionWorkspaceMetadataForPersistence } from "../services/heartbeat/workspaces.js"; import { describeGitExpirationFailure, expireReconstructableGitWorkspace } from "../services/git-workspace-expiration.js"; +import { nestedWorktreeBranchPrefix, restoreNestedRepositoryWorktree, withNestedRepositoryBaseLock } from "../services/nested-repository-worktrees.js"; import * as activityLog from "../services/activity-log.js"; import { readGitCacheRestoration } from "../services/git-workspace-restoration.js"; -import { gitCacheExpirationLockLocations, readGitCacheExpiration } from "../services/git-workspace-restoration.js"; +import { assertReproducibleManagedCheckout, gitCacheExpirationLockLocations, readGitCacheExpiration } from "../services/git-workspace-restoration.js"; const gitEnv = { ...process.env, GIT_AUTHOR_NAME: "Fixture", GIT_AUTHOR_EMAIL: "fixture@example.test", GIT_COMMITTER_NAME: "Fixture", GIT_COMMITTER_EMAIL: "fixture@example.test", GIT_TERMINAL_PROMPT: "0" }; @@ -153,7 +154,7 @@ describeEmbeddedPostgres("reconstructable Git expiration", () => { expect(await fs.readFile(settingsPath, "utf8")).toBe("later task settings\n"); }); - async function nestedFixture(managed = true) { + async function nestedFixture(managed = true, shared = false) { const f = await fixture(managed, !managed, true); const nestedSource = path.join(path.dirname(f.source), "nested-author"); const nestedRemote = path.join(path.dirname(f.source), "nested.git"); @@ -166,7 +167,8 @@ describeEmbeddedPostgres("reconstructable Git expiration", () => { git(path.dirname(nestedSource), "clone", "--bare", nestedSource, nestedRemote); const [source] = await db.insert(projectWorkspaces).values({ companyId: f.companyId, projectId: f.projectId!, name: "Nested repository", repoUrl: pathToFileURL(nestedRemote).href, repoRef: "main" }).returning(); - const preparation = { cwd: f.cwd, anchorRepoUrl: f.base.repoUrl, workspaces: [source!], resolveGitAuth: async () => null }; + const preparation = { cwd: f.cwd, anchorRepoUrl: f.base.repoUrl, workspaces: [source!], resolveGitAuth: async () => null, + ...(shared ? { sharedBase: { companyId: f.companyId, projectId: f.projectId! } } : {}) }; const [nested] = await prepareProjectRepositoryWorkspaces(preparation); const settings = Buffer.from('{"permissions":{"allow":["Read(/user/path)"]},"env":{"LOCAL_SETTING":"preserve-me"}}\n'); const settingsPath = path.join(f.cwd, ".claude/settings.local.json"); @@ -175,7 +177,8 @@ describeEmbeddedPostgres("reconstructable Git expiration", () => { const nestedSettingsPath = path.join(nested!.cwd, ".claude/settings.local.json"); await fs.mkdir(path.dirname(nestedSettingsPath)); await fs.writeFile(nestedSettingsPath, settings, { mode: 0o640 }); - return { ...f, nestedSource, nestedRemote, nestedA, preparation, nested: nested!, settings, settingsPath, nestedSettingsPath }; + return { ...f, nestedSource, nestedRemote, nestedA, preparation, nested: nested!, settings, settingsPath, nestedSettingsPath, + sharedBase: resolveManagedProjectWorkspaceDir({ companyId: f.companyId, projectId: f.projectId!, repoName: "nested" }) }; } it("expires a nested workspace and its unused private base with the generated ignore entry", async () => { @@ -454,7 +457,7 @@ describeEmbeddedPostgres("reconstructable Git expiration", () => { expect(await f.expire({ beforeDelete: async () => { throw new Error("stop_before_locks"); } })) .toMatchObject({ expired: false, skipReason: "stop_before_locks" }); const row = await f.row(); - const locations = gitCacheExpirationLockLocations(row); + const locations = await gitCacheExpirationLockLocations(row); const token = readGitCacheExpiration(row)!.lockToken!; const module = new URL("../services/git-workspace-expiration-locks.ts", import.meta.url).href; const child = spawnSync(process.execPath, ["--import", createRequire(import.meta.url).resolve("tsx"), "--input-type=module", "-e", ` @@ -996,4 +999,365 @@ describeEmbeddedPostgres("reconstructable Git expiration", () => { expect(git(f.cwd, "rev-parse", "HEAD")).toBe(f.commitA); }); + + describe("nested checkouts as worktrees of a shared base", () => { + const restorePoint = () => ({ storage: getStorageService(), notBefore: new Date(Date.now() - 24 * 3_600_000), keyDurable: true }); + const worktrees = (base: string) => git(base, "worktree", "list", "--porcelain").split("\n") + .filter(line => line.startsWith("worktree ")).map(line => line.slice(9)); + const real = (target: string) => fs.realpath(target); + async function sharedFixture() { + const f = await nestedFixture(false, true); + const siblingCwd = path.join(path.dirname(f.source), "sibling-task"); + await fs.mkdir(siblingCwd); + const [sibling] = await prepareProjectRepositoryWorkspaces({ ...f.preparation, cwd: siblingCwd }); + return { ...f, siblingCwd, sibling: sibling! }; + } + + it("prepares nested checkouts of two tasks as worktrees of one shared clone", async () => { + const f = await sharedFixture(); + const base = await real(f.sharedBase); + for (const checkout of [f.nested.cwd, f.sibling.cwd]) { + expect((await fs.lstat(path.join(checkout, ".git"))).isFile()).toBe(true); + expect(git(checkout, "rev-parse", "--path-format=absolute", "--git-common-dir")).toBe(path.join(base, ".git")); + expect(git(checkout, "rev-parse", "HEAD")).toBe(f.nestedA); + expect(git(checkout, "remote", "get-url", "origin")).toBe(pathToFileURL(f.nestedRemote).href); + expect(git(checkout, "symbolic-ref", "--short", "HEAD")).toMatch(/^paperclip\/[a-f0-9]{12}\/main$/); + expect(git(checkout, "rev-parse", "--abbrev-ref", "@{upstream}")).toBe("origin/main"); + } + expect(git(f.nested.cwd, "symbolic-ref", "HEAD")).not.toBe(git(f.sibling.cwd, "symbolic-ref", "HEAD")); + expect(await worktrees(base)).toEqual(expect.arrayContaining([await real(f.nested.cwd), await real(f.sibling.cwd)])); + expect(await prepareProjectRepositoryWorkspaces(f.preparation)).toMatchObject([{ cwd: f.nested.cwd }]); + expect(git(f.nested.cwd, "rev-parse", "HEAD")).toBe(f.nestedA); + }); + + it("serializes concurrent preparation of tasks sharing one base", async () => { + const f = await nestedFixture(false, true); + const tasks = await Promise.all([1, 2, 3].map(async index => { + const cwd = path.join(path.dirname(f.source), `concurrent-${index}`); + await fs.mkdir(cwd); + return cwd; + })); + const prepared = await Promise.all(tasks.map(cwd => prepareProjectRepositoryWorkspaces({ ...f.preparation, cwd }))); + const base = await real(f.sharedBase); + for (const [repository] of prepared) { + expect(git(repository!.cwd, "rev-parse", "--path-format=absolute", "--git-common-dir")).toBe(path.join(base, ".git")); + } + expect(await worktrees(base)).toEqual(expect.arrayContaining(await Promise.all(prepared.map(([repository]) => real(repository!.cwd))))); + const waiting = path.join(path.dirname(f.source), "waiting-task"); + await fs.mkdir(waiting); + let release!: () => void; + const held = withNestedRepositoryBaseLock(f.sharedBase, () => new Promise(resolve => { release = resolve; })); + const pending = prepareProjectRepositoryWorkspaces({ ...f.preparation, cwd: waiting }); + await new Promise(resolve => setTimeout(resolve, 500)); + expect(await exists(path.join(waiting, ".paperclip-repositories", path.basename(f.nested.cwd)))).toBe(false); + release(); + await held; + const [late] = await pending; + expect(git(late!.cwd, "rev-parse", "--path-format=absolute", "--git-common-dir")).toBe(path.join(base, ".git")); + }); + + it("tracks a dotted branch ref and proves its tracking configuration", async () => { + const f = await nestedFixture(false, true); + git(f.nestedSource, "push", "-q", f.nestedRemote, "main:release-1.0"); + const task = path.join(path.dirname(f.source), "dotted-task"); + await fs.mkdir(task); + const [prepared] = await prepareProjectRepositoryWorkspaces({ ...f.preparation, cwd: task, + workspaces: [{ ...f.preparation.workspaces[0]!, repoRef: "release-1.0" }] }); + expect(git(prepared!.cwd, "symbolic-ref", "--short", "HEAD")).toMatch(/^paperclip\/[a-f0-9]{12}\/release-1\.0$/); + expect(git(prepared!.cwd, "rev-parse", "--abbrev-ref", "@{upstream}")).toBe("origin/release-1.0"); + git(prepared!.cwd, "pull", "-q"); + await expect(assertReproducibleManagedCheckout(prepared!.cwd, { repoUrl: pathToFileURL(f.nestedRemote).href, + resolveGitAuth: async () => null, baseCandidates: [f.sharedBase] })).resolves.toMatchObject({ headOnRemote: true }); + }); + + it("prepares a new branch when a retained checkout still holds the original one", async () => { + const f = await nestedFixture(false, true); + git(f.nestedSource, "push", "-q", f.nestedRemote, "main:other"); + const original = git(f.nested.cwd, "symbolic-ref", "HEAD"); + await prepareProjectRepositoryWorkspaces({ ...f.preparation, workspaces: [{ ...f.preparation.workspaces[0]!, repoRef: "other" }] }); + const detachedRoot = path.join(f.cwd, ".paperclip-runtime", "detached-repositories"); + const [retained] = await fs.readdir(detachedRoot); + expect(git(path.join(detachedRoot, retained!), "symbolic-ref", "HEAD")).toBe(original); + const [back] = await prepareProjectRepositoryWorkspaces(f.preparation); + expect(back!.cwd).toBe(f.nested.cwd); + expect(git(back!.cwd, "symbolic-ref", "HEAD")).toBe(original.replace(/\/main$/, "/r2/main")); + expect(git(back!.cwd, "rev-parse", "--abbrev-ref", "@{upstream}")).toBe("origin/main"); + expect(git(path.join(detachedRoot, retained!), "symbolic-ref", "HEAD")).toBe(original); + }); + + it("follows a changed remote default branch for new tasks", async () => { + const f = await nestedFixture(false, true); + git(f.nestedSource, "push", "-q", f.nestedRemote, "main:trunk"); + git(f.nestedRemote, "symbolic-ref", "HEAD", "refs/heads/trunk"); + const task = path.join(path.dirname(f.source), "default-branch-task"); + await fs.mkdir(task); + const [prepared] = await prepareProjectRepositoryWorkspaces({ ...f.preparation, cwd: task, + workspaces: [{ ...f.preparation.workspaces[0]!, repoRef: null }] }); + expect(git(prepared!.cwd, "symbolic-ref", "--short", "HEAD")).toMatch(/^paperclip\/[a-f0-9]{12}\/trunk$/); + expect(git(prepared!.cwd, "rev-parse", "--abbrev-ref", "@{upstream}")).toBe("origin/trunk"); + }); + + it("restores a renumbered owned branch tracking its source branch, not a same-named remote branch", async () => { + const f = await nestedFixture(false, true); + git(f.nestedSource, "commit", "-q", "--allow-empty", "-m", "Unrelated r2 branch"); + git(f.nestedSource, "push", "-q", f.nestedRemote, "HEAD:r2/main"); + const base = await real(f.sharedBase); + git(base, "fetch", "-q", "origin"); + const target = path.join(path.dirname(f.source), "renumbered-task", "checkout"); + await fs.mkdir(path.dirname(target)); + const branchName = `${await nestedWorktreeBranchPrefix(target)}r2/main`; + await restoreNestedRepositoryWorktree({ base, cwd: target, commit: f.nestedA, branchName, repoRef: null, afterAdd: async () => undefined }); + expect(git(target, "symbolic-ref", "--short", "HEAD")).toBe(branchName); + expect(git(target, "rev-parse", "--abbrev-ref", "@{upstream}")).toBe("origin/main"); + }); + + it("starts a fully qualified branch ref at the current remote commit and rejects a deleted branch", async () => { + const f = await nestedFixture(false, true); + await fs.writeFile(path.join(f.nestedSource, "nested.txt"), "nested B\n"); + git(f.nestedSource, "commit", "-qam", "Nested B"); + git(f.nestedSource, "push", "-q", f.nestedRemote, "main", "main:short-lived"); + const nestedB = git(f.nestedSource, "rev-parse", "HEAD"); + const prepare = async (name: string, repoRef: string) => { + const task = path.join(path.dirname(f.source), name); + await fs.mkdir(task); + return prepareProjectRepositoryWorkspaces({ ...f.preparation, cwd: task, workspaces: [{ ...f.preparation.workspaces[0]!, repoRef }] }); + }; + const [qualified] = await prepare("qualified-task", "refs/heads/main"); + expect(git(qualified!.cwd, "rev-parse", "HEAD")).toBe(nestedB); + expect(git(qualified!.cwd, "rev-parse", "--abbrev-ref", "@{upstream}")).toBe("origin/main"); + await prepare("short-lived-task", "short-lived"); + git(f.nestedRemote, "branch", "-D", "short-lived"); + await expect(prepare("deleted-branch-task", "short-lived")).rejects.toThrow('Repository ref "short-lived" was not found'); + }); + + it("never runs hooks planted in the shared clone while preparing or restoring", async () => { + const f = await nestedFixture(false, true); + const marker = path.join(path.dirname(f.source), "hook-ran"); + const hooks = path.join(await real(f.sharedBase), ".git", "hooks"); + for (const hook of ["post-checkout", "reference-transaction"]) { + await fs.writeFile(path.join(hooks, hook), `#!/bin/sh\necho ${hook} >> ${JSON.stringify(marker)}\n`, { mode: 0o755 }); + } + const task = path.join(path.dirname(f.source), "hooked-task"); + await fs.mkdir(task); + await prepareProjectRepositoryWorkspaces({ ...f.preparation, cwd: task }); + const target = path.join(path.dirname(f.source), "hooked-restore", "checkout"); + await fs.mkdir(path.dirname(target)); + await restoreNestedRepositoryWorktree({ base: await real(f.sharedBase), cwd: target, commit: f.nestedA, + branchName: `${await nestedWorktreeBranchPrefix(target)}main`, repoRef: "main", preserved: [f.nestedA], afterAdd: async () => undefined }); + expect(await exists(marker)).toBe(false); + }); + + it("keeps adopting a legacy standalone nested clone", async () => { + const f = await nestedFixture(false); + const [prepared] = await prepareProjectRepositoryWorkspaces({ ...f.preparation, sharedBase: { companyId: f.companyId, projectId: f.projectId! } }); + expect(prepared!.cwd).toBe(f.nested.cwd); + expect((await fs.lstat(path.join(f.nested.cwd, ".git"))).isDirectory()).toBe(true); + expect(await exists(f.sharedBase)).toBe(false); + }); + + it("expires a clean nested worktree, unregisters only it, and restores it from the base", async () => { + const f = await sharedFixture(); + const base = await real(f.sharedBase); + const ownBranch = git(f.nested.cwd, "symbolic-ref", "HEAD"); + const siblingBranch = git(f.sibling.cwd, "symbolic-ref", "HEAD"); + const nestedPath = await real(f.nested.cwd); + expect(await f.expire({ expireBase: false })).toMatchObject({ expired: true, skipReason: null }); + expect(await worktrees(base)).not.toContain(nestedPath); + expect(await worktrees(base)).toContain(await real(f.sibling.cwd)); + expect(() => git(base, "rev-parse", "--verify", "-q", ownBranch)).toThrow(); + expect(git(base, "rev-parse", siblingBranch)).toBe(f.nestedA); + const recipe = (await f.row()).metadata?.gitCacheRestoration as Record; + expect(recipe.contents.checkouts[0]).toMatchObject({ commit: f.nestedA, branchName: ownBranch.slice("refs/heads/".length), + worktree: { base, admin: expect.any(String) } }); + expect(await f.reopen()).toMatchObject({ ok: true, reopened: true }); + expect((await fs.lstat(path.join(f.nested.cwd, ".git"))).isFile()).toBe(true); + expect(git(f.nested.cwd, "rev-parse", "--path-format=absolute", "--git-common-dir")).toBe(path.join(base, ".git")); + expect(git(f.nested.cwd, "symbolic-ref", "HEAD")).toBe(ownBranch); + expect(git(f.nested.cwd, "rev-parse", "HEAD")).toBe(f.nestedA); + expect(git(f.nested.cwd, "rev-parse", "--abbrev-ref", "@{upstream}")).toBe("origin/main"); + expect(await fs.readFile(f.nestedSettingsPath)).toEqual(f.settings); + expect(await prepareProjectRepositoryWorkspaces(f.preparation)).toMatchObject([{ cwd: f.nested.cwd }]); + }); + + it("recreates a deleted shared base when restoring an expired nested worktree", async () => { + const f = await nestedFixture(false, true); + expect(await f.expire({ expireBase: false })).toMatchObject({ expired: true }); + await fs.rm(f.sharedBase, { recursive: true, force: true }); + expect(await f.reopen()).toMatchObject({ ok: true, reopened: true }); + expect(git(f.nested.cwd, "rev-parse", "--path-format=absolute", "--git-common-dir")).toBe(path.join(await real(f.sharedBase), ".git")); + expect(git(f.nested.cwd, "rev-parse", "HEAD")).toBe(f.nestedA); + }); + + it("blocks unpushed nested work and saves it in a restore point", async () => { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "31".repeat(32)); + const f = await sharedFixture(); + const branch = git(f.nested.cwd, "symbolic-ref", "HEAD"); + await fs.writeFile(path.join(f.nested.cwd, "nested.txt"), "nested B (unpushed)\n"); + git(f.nested.cwd, "commit", "-qam", "Nested B unpushed"); + const unpushed = git(f.nested.cwd, "rev-parse", "HEAD"); + await fs.writeFile(path.join(f.nested.cwd, "draft.md"), "nested untracked work\n"); + expect(await f.expire({ expireBase: false })).toMatchObject({ expired: false, skipReason: "dirty_or_untracked_files" }); + await fs.rename(path.join(f.nested.cwd, "draft.md"), path.join(f.siblingCwd, "draft.md")); + expect(await f.expire({ expireBase: false })).toMatchObject({ expired: false, skipReason: "unpushed_managed_checkout" }); + await fs.rename(path.join(f.siblingCwd, "draft.md"), path.join(f.nested.cwd, "draft.md")); + expect(await f.expire({ expireBase: false, restorePoint: restorePoint() })).toMatchObject({ expired: true, restorePoint: true, skipReason: null }); + expect(await f.reopen()).toMatchObject({ ok: true, reopened: true }); + expect(git(f.nested.cwd, "symbolic-ref", "HEAD")).toBe(branch); + expect(git(f.nested.cwd, "rev-parse", "HEAD")).toBe(unpushed); + expect(await fs.readFile(path.join(f.nested.cwd, "draft.md"), "utf8")).toBe("nested untracked work\n"); + expect(git(f.sibling.cwd, "rev-parse", "HEAD")).toBe(f.nestedA); + vi.unstubAllEnvs(); + }); + + it("saves history kept only in the nested worktree's reflog and restores it there", async () => { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "31".repeat(32)); + const f = await sharedFixture(); + await fs.writeFile(path.join(f.nested.cwd, "nested.txt"), "rewritten away\n"); + git(f.nested.cwd, "commit", "-qam", "Only in the reflog"); + const orphan = git(f.nested.cwd, "rev-parse", "HEAD"); + git(f.nested.cwd, "reset", "-q", "--hard", "HEAD~1"); + expect(await f.expire({ expireBase: false })).toMatchObject({ expired: false, skipReason: "unique_original_head" }); + expect(await f.expire({ expireBase: false, restorePoint: restorePoint() })).toMatchObject({ expired: true, restorePoint: true }); + git(await real(f.sharedBase), "reflog", "expire", "--expire=now", "--all"); + git(await real(f.sharedBase), "gc", "-q", "--prune=now"); + expect(await f.reopen()).toMatchObject({ ok: true, reopened: true }); + expect(git(f.nested.cwd, "rev-parse", "HEAD")).toBe(f.nestedA); + expect(git(f.nested.cwd, "reflog", "show", "--format=%H", "HEAD").split("\n")).toContain(orphan); + expect(git(f.nested.cwd, "cat-file", "-t", orphan)).toBe("commit"); + await expect(assertReproducibleManagedCheckout(f.nested.cwd, { repoUrl: pathToFileURL(f.nestedRemote).href, + resolveGitAuth: async () => null, baseCandidates: [f.sharedBase] })).rejects.toThrow("unique_reflog_commits"); + vi.unstubAllEnvs(); + }); + + it("blocks on stashes, unowned checked-out branches, and unpushed owned branches", async () => { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "31".repeat(32)); + const f = await sharedFixture(); + await fs.writeFile(path.join(f.nested.cwd, "nested.txt"), "stashed work\n"); + git(f.nested.cwd, "stash", "-q"); + expect(await f.expire({ expireBase: false, restorePoint: restorePoint() })).toMatchObject({ expired: false, skipReason: "unsupported_local_git_state" }); + git(f.nested.cwd, "stash", "pop", "-q"); + git(f.nested.cwd, "checkout", "-q", "--", "nested.txt"); + const owned = git(f.nested.cwd, "symbolic-ref", "--short", "HEAD"); + git(f.nested.cwd, "checkout", "-q", "-b", "side"); + expect(await f.expire({ expireBase: false, restorePoint: restorePoint() })).toMatchObject({ expired: false, skipReason: "unowned_nested_worktree_branch" }); + git(f.nested.cwd, "checkout", "-q", owned); + git(f.nested.cwd, "checkout", "-q", "-b", `${owned}-extra`); + git(f.nested.cwd, "commit", "-q", "--allow-empty", "-m", "Unpushed extra owned branch"); + git(f.nested.cwd, "checkout", "-q", owned); + await fs.writeFile(path.join(f.nested.cwd, "draft.md"), "untracked\n"); + expect(await f.expire({ expireBase: false, restorePoint: restorePoint() })).toMatchObject({ expired: false, skipReason: "unpushed_managed_checkout_branches" }); + expect(await exists(f.cwd)).toBe(true); + vi.unstubAllEnvs(); + }); + + it("blocks an unpushed owned branch or owned reflog entry outside HEAD history", async () => { + const f = await sharedFixture(); + const owned = git(f.nested.cwd, "symbolic-ref", "--short", "HEAD"); + const orphan = execFileSync("git", ["commit-tree", git(f.nested.cwd, "rev-parse", "HEAD^{tree}"), "-p", "HEAD", "-m", "Owned but unpushed"], + { cwd: f.nested.cwd, env: gitEnv, encoding: "utf8" }).trim(); + git(f.nested.cwd, "update-ref", "--create-reflog", `refs/heads/${owned}-extra`, orphan); + expect(await f.expire({ expireBase: false })).toMatchObject({ expired: false, skipReason: "unpushed_managed_checkout" }); + git(f.nested.cwd, "update-ref", `refs/heads/${owned}-extra`, f.nestedA); + expect(await f.expire({ expireBase: false })).toMatchObject({ expired: false, skipReason: "unpushed_managed_checkout" }); + git(f.nested.cwd, "update-ref", "-d", `refs/heads/${owned}-extra`); + expect(await f.expire({ expireBase: false })).toMatchObject({ expired: true, skipReason: null }); + }); + + it("rejects too many owned branches before reading their reflogs", async () => { + const f = await nestedFixture(false, true); + const owned = git(f.nested.cwd, "symbolic-ref", "--short", "HEAD"); + const refs = Array.from({ length: 65 }, (_, index) => `create refs/heads/${owned}-many-${index} ${f.nestedA}`).join("\n"); + execFileSync("git", ["update-ref", "--stdin"], { cwd: f.nested.cwd, env: gitEnv, input: `${refs}\n` }); + expect(await f.expire({ expireBase: false })).toMatchObject({ expired: false, skipReason: "managed_checkout_local_tip_budget" }); + }); + + it.each([ + ["a named pipe", async (file: string) => { execFileSync("mkfifo", [file]); }], + ["an oversized file", async (file: string) => { await fs.writeFile(file, "x".repeat(8 * 1024)); }], + ["a symlink", async (file: string) => { await fs.symlink("/dev/zero", file); }], + ])("rejects a nested worktree whose administration pointer is %s without reading it unsafely", async (_name, plant) => { + const f = await nestedFixture(false, true); + const admin = git(f.nested.cwd, "rev-parse", "--absolute-git-dir"); + const pointer = path.join(admin, "gitdir"); + await fs.rm(pointer); + await plant(pointer); + expect(await f.expire({ expireBase: false })).toMatchObject({ expired: false, skipReason: "unproven_managed_checkout" }); + await expect(prepareProjectRepositoryWorkspaces(f.preparation)).rejects.toThrow("Invalid project repository checkout path"); + }); + + it("ignores another task's unpushed branch and keeps it after expiring this worktree", async () => { + const f = await sharedFixture(); + git(f.sibling.cwd, "commit", "-q", "--allow-empty", "-m", "Sibling unpushed work"); + const siblingWork = git(f.sibling.cwd, "rev-parse", "HEAD"); + const siblingBranch = git(f.sibling.cwd, "symbolic-ref", "HEAD"); + git(f.nested.cwd, "branch", "agent-shared-branch", siblingWork); + expect(await f.expire({ expireBase: false })).toMatchObject({ expired: true, skipReason: null }); + const base = await real(f.sharedBase); + expect(git(base, "rev-parse", siblingBranch)).toBe(siblingWork); + expect(git(base, "rev-parse", "refs/heads/agent-shared-branch")).toBe(siblingWork); + expect(git(f.sibling.cwd, "rev-parse", "HEAD")).toBe(siblingWork); + }); + + it("rejects a nested worktree whose base is not the project's shared clone", async () => { + const f = await sharedFixture(); + const foreign = path.join(path.dirname(f.source), "foreign-base"); + git(path.dirname(f.source), "clone", "-q", pathToFileURL(f.nestedRemote).href, foreign); + git(f.nested.cwd, "checkout", "-q", "--detach"); + const ownBranch = git(f.sibling.cwd, "symbolic-ref", "HEAD"); + await fs.rm(f.nested.cwd, { recursive: true, force: true }); + git(await real(f.sharedBase), "worktree", "prune"); + git(foreign, "worktree", "add", "-q", "--detach", f.nested.cwd); + await expect(prepareProjectRepositoryWorkspaces(f.preparation)).rejects.toThrow("Invalid project repository checkout path"); + await expect(assertReproducibleManagedCheckout(f.nested.cwd, { repoUrl: pathToFileURL(f.nestedRemote).href, + resolveGitAuth: async () => null, baseCandidates: [f.sharedBase] })).rejects.toThrow("unproven_managed_checkout"); + expect(await f.expire({ expireBase: false })).toMatchObject({ expired: false, skipReason: "unproven_managed_checkout" }); + expect(git(f.sibling.cwd, "symbolic-ref", "HEAD")).toBe(ownBranch); + }); + + it("recovers process-abandoned nested worktree locks held in the shared base", async () => { + const f = await nestedFixture(true, true); + const locations = await crashWithExpirationLocks(f); + const base = await real(f.sharedBase); + const nestedLocks = locations.filter(location => location.checkout !== null); + expect(nestedLocks.map(location => location.relativePath)).toEqual(expect.arrayContaining([ + expect.stringMatching(/^\.git\/worktrees\/[^/]+\/index\.lock$/), expect.stringMatching(/^\.git\/refs\/heads\/paperclip\/[a-f0-9]{12}\/main\.lock$/)])); + for (const location of nestedLocks) { + expect(await real(location.root)).toBe(base); + expect(await exists(path.join(location.root, location.relativePath))).toBe(true); + } + expect(await f.expire()).toMatchObject({ expired: true }); + for (const location of nestedLocks) expect(await exists(path.join(location.root, location.relativePath))).toBe(false); + expect(await f.reopen()).toMatchObject({ ok: true, reopened: true }); + expect(git(f.nested.cwd, "rev-parse", "HEAD")).toBe(f.nestedA); + expect(git(f.nested.cwd, "rev-parse", "--path-format=absolute", "--git-common-dir")).toBe(path.join(base, ".git")); + }); + + it("removes only the deleted task's nested worktrees and keeps its unpushed branch", async () => { + const f = await sharedFixture(); + const { cleanupExecutionWorkspaceArtifacts } = await import("../services/workspace-runtime.js"); + const base = await real(f.sharedBase); + git(f.nested.cwd, "commit", "-q", "--allow-empty", "-m", "Unpushed before deletion"); + const unpushed = git(f.nested.cwd, "rev-parse", "HEAD"); + const ownBranch = git(f.nested.cwd, "symbolic-ref", "HEAD"); + const siblingBranch = git(f.sibling.cwd, "symbolic-ref", "HEAD"); + const nestedPath = await real(f.nested.cwd); + const row = await f.row(); + const cleanup = await cleanupExecutionWorkspaceArtifacts({ workspace: { ...row, providerType: "git_worktree", metadata: {} }, + runCleanupCommands: false }); + expect(cleanup).toMatchObject({ cleaned: true, warnings: [] }); + expect(await exists(f.cwd)).toBe(false); + expect(await worktrees(base)).not.toContain(nestedPath); + expect(await worktrees(base)).toContain(await real(f.sibling.cwd)); + expect(git(base, "rev-parse", ownBranch)).toBe(unpushed); + const siblingPath = await real(f.sibling.cwd); + const siblingWorkspace = { ...row, id: randomUUID(), cwd: f.siblingCwd, providerRef: f.siblingCwd, providerType: "local_fs", + metadata: { createdByRuntime: true } }; + expect(await cleanupExecutionWorkspaceArtifacts({ workspace: siblingWorkspace, runCleanupCommands: false })) + .toMatchObject({ cleaned: true, warnings: [] }); + expect(await worktrees(base)).not.toContain(siblingPath); + expect(() => git(base, "rev-parse", "--verify", "-q", siblingBranch)).toThrow(); + expect(git(base, "rev-parse", ownBranch)).toBe(unpushed); + }); + }); + }); diff --git a/server/src/__tests__/heartbeat-project-repositories.test.ts b/server/src/__tests__/heartbeat-project-repositories.test.ts index 6a46a86c773..0ad7ce0d33c 100644 --- a/server/src/__tests__/heartbeat-project-repositories.test.ts +++ b/server/src/__tests__/heartbeat-project-repositories.test.ts @@ -797,6 +797,11 @@ suite("task project repository provisioning", () => { const expectedBytes = scenario === "fresh" || urlOnly ? "PRIVATE_DIRTY_SIBLING_BYTES" : "PRESERVED_EXISTING_SIBLING_BYTES"; expect(await readFile(path.join(copy, "private.txt"), "utf8")).toBe(expectedBytes); if (urlOnly) await expect(readFile(path.join(copy, "uncommitted.txt"), "utf8")).rejects.toMatchObject({ code: "ENOENT" }); + if (urlOnly) { + const sharedBase = await realpath(resolveManagedProjectWorkspaceDir({ companyId, projectId, repoName: scenario === "absolute_path" ? null : "private-sibling" })); + expect(execFileSync("git", ["rev-parse", "--path-format=absolute", "--git-common-dir"], { cwd: copy, encoding: "utf8" }).trim()).toBe(path.join(sharedBase, ".git")); + expect(execFileSync("git", ["symbolic-ref", "--short", "HEAD"], { cwd: copy, encoding: "utf8" }).trim()).toMatch(/^paperclip\/[a-f0-9]{12}\/main$/); + } expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0].executionWorkspaceId).toBe(binding); const app = express(); app.use(express.json()); @@ -823,6 +828,10 @@ suite("task project repository provisioning", () => { expect(detached.length).toBeGreaterThan(0); const retained = await Promise.all(detached.map(name => readFile(path.join(detachedRoot, name, "private.txt"), "utf8").catch(() => null))); expect(retained).toContain(expectedBytes); + for (const name of scenario === "occupied" ? [] : detached) { + expect(execFileSync("git", ["rev-parse", "--show-toplevel"], { cwd: path.join(detachedRoot, name), encoding: "utf8" }).trim()) + .toBe(await realpath(path.join(detachedRoot, name))); + } } finally { vi.stubEnv("PAPERCLIP_ISSUE_PRIVACY_MODE", previousPrivacyMode); } diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index 6fa75220605..8d490f35cbc 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -4011,7 +4011,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic eq(projectWorkspaces.companyId, row.companyId), eq(projectWorkspaces.projectId, row.projectId!), )).orderBy(asc(projectWorkspaces.createdAt), asc(projectWorkspaces.id)); await prepareProjectRepositoryWorkspaces({ cwd: targetCwd, anchorRepoUrl: row.repoUrl, workspaces: repositoryRows, - resolveGitAuth: gitAuth, + sharedBase: { companyId: row.companyId, projectId: row.projectId! }, resolveGitAuth: gitAuth, prepareLocalSource: async sourceCwd => { const sourceIdentity = await fs.realpath(sourceCwd); return async () => { diff --git a/server/src/services/git-workspace-contents.ts b/server/src/services/git-workspace-contents.ts index 4eb70d80324..e0f7802da6e 100644 --- a/server/src/services/git-workspace-contents.ts +++ b/server/src/services/git-workspace-contents.ts @@ -11,7 +11,9 @@ import { localRepositoryPath } from "./local-repository-path.js"; import { managedProjectCheckoutName } from "./managed-project-checkout-name.js"; import { materializeManagedProjectWorkspace } from "./managed-repository-checkout.js"; import { ensureSavedGitCommit, gitCacheCommand, GitCacheRestorationFailure } from "./git-workspace-restoration.js"; -import { applyGitRestorePointToWorktree, ensureGitRestorePointCommit, type GitRestorePointRef } from "./git-restore-point-storage.js"; +import { applyGitRestorePointToWorktree, ensureGitRestorePointCommit, loadGitRestorePoint, type GitRestorePointRef } from "./git-restore-point-storage.js"; +import { importRestorePoint } from "./git-restore-point.js"; +import { isNestedWorktreeBranchName, isVerifiedNestedRepositoryBase, nestedRepositoryBaseCandidates, readLinkedNestedCheckout, recordedNestedRepositoryBase, restoreNestedRepositoryWorktree, withNestedRepositoryBaseLock } from "./nested-repository-worktrees.js"; import { getStorageService } from "../storage/index.js"; import type { StorageService } from "../storage/types.js"; @@ -23,6 +25,7 @@ type Settings = { material: StoredSecretVersionMaterial } | null; export type ManagedCheckoutSource = { id: string; repoUrl: string; repoRef: string | null; cwd: string | null }; export type ManagedCheckoutContents = { name: string; source: ManagedCheckoutSource; commit: string; branchName: string | null; + worktree?: { base: string; admin: string }; restorePoint?: GitRestorePointRef; }; export type GitWorkspaceContents = { @@ -108,7 +111,12 @@ export function validateGitWorkspaceContents(contents: GitWorkspaceContents | un || !/^[a-f0-9]{40}$/.test(checkout.commit) || !(checkout.branchName === null || typeof checkout.branchName === "string" && checkout.branchName.length > 0 && checkout.branchName.length <= 1024 && !checkout.branchName.startsWith("-") && !checkout.branchName.includes("\0")) - || !settingsValid(checkout.settings)) throw new GitCacheRestorationFailure(); + || !settingsValid(checkout.settings) + || (checkout.worktree !== undefined && (!checkout.worktree || typeof checkout.worktree.base !== "string" + || !path.isAbsolute(checkout.worktree.base) || checkout.worktree.base.includes("\0") || checkout.worktree.base.length > 4096 + || typeof checkout.worktree.admin !== "string" || !/^[A-Za-z0-9._-]{1,255}$/.test(checkout.worktree.admin) + || [".", ".."].includes(checkout.worktree.admin) || Object.keys(checkout.worktree).length !== 2 + || (checkout.branchName !== null && !isNestedWorktreeBranchName(checkout.branchName))))) throw new GitCacheRestorationFailure(); names.add(checkout.name); } } @@ -127,7 +135,8 @@ export function selectRestoredProjectRepositories { const commit = (await gitCacheCommand(checkout, ["rev-parse", "HEAD"], signal)).trim(); const branchName = (await gitCacheCommand(checkout, ["symbolic-ref", "--quiet", "--short", "HEAD"], signal).catch(() => "")).trim() || null; - return { name, source, commit, branchName }; + const linked = await readLinkedNestedCheckout(checkout); + return linked ? { name, source, commit, branchName, worktree: { base: linked.base, admin: linked.admin } } : { name, source, commit, branchName }; } export async function captureGitWorkspaceContents(cwd: string, identity: Identity, checkouts: ManagedCheckoutContents[]): Promise { @@ -191,7 +200,7 @@ async function restoreSettings(cwd: string, settings: Settings, identity: Identi } finally { await fs.rm(temporary, { force: true }); } } -export async function restoreGitWorkspaceContents(input: Identity & { cwd: string; contents?: GitWorkspaceContents; +export async function restoreGitWorkspaceContents(input: Identity & { cwd: string; contents?: GitWorkspaceContents; projectId?: string | null; prepareLocalSource?: (sourceCwd: string) => Promise<() => Promise>; storage?: StorageService; resolveGitAuth: GitRemoteAuthProvider; authorizeSource: (source: ManagedCheckoutSource) => Promise }) { @@ -208,9 +217,19 @@ export async function restoreGitWorkspaceContents(input: Identity & { cwd: strin const destination = path.join(root!, checkout.name); const relative = `${PROJECT_REPOSITORIES_DIR}/${checkout.name}/${CLAUDE_SETTINGS_PATH}`; const existing = await entry(destination); - if (existing) { + const candidates = checkout.worktree && input.projectId + ? nestedRepositoryBaseCandidates({ companyId: input.companyId, projectId: input.projectId, repoUrl: checkout.source.repoUrl }) : null; + if (checkout.worktree && !candidates) throw new GitCacheRestorationFailure(); + if (existing && checkout.worktree) { + const linked = existing.isDirectory() && !existing.isSymbolicLink() ? await readLinkedNestedCheckout(destination, candidates!) : null; + if (!linked || linked.checkout !== destination || !await isVerifiedNestedRepositoryBase(linked.base, candidates!, checkout.source.repoUrl)) { + throw new GitCacheRestorationFailure(); + } + } else if (existing) { if (!existing.isDirectory() || existing.isSymbolicLink() || !(await entry(path.join(destination, ".git")))?.isDirectory() || (await fs.realpath(path.join(destination, ".git"))) !== path.join(destination, ".git")) throw new GitCacheRestorationFailure(); + } + if (existing) { if ((await gitCacheCommand(destination, ["rev-parse", "--show-toplevel"])).trim() !== destination || (await gitCacheCommand(destination, ["config", "--local", "--null", "--get-all", "remote.origin.url"])) !== `${checkout.source.repoUrl}\0`) throw new GitCacheRestorationFailure(); const actual = await readManagedCheckoutContents(destination, checkout.name, checkout.source); @@ -220,6 +239,36 @@ export async function restoreGitWorkspaceContents(input: Identity & { cwd: strin await restoreSettings(destination, checkout.settings, input, relative); continue; } + if (checkout.worktree) { + const recordedBase = await recordedNestedRepositoryBase(checkout.worktree.base, candidates!); + if (!recordedBase) throw new GitCacheRestorationFailure(); + if (!await isVerifiedNestedRepositoryBase(recordedBase, candidates!, checkout.source.repoUrl)) { + if (await entry(recordedBase)) throw new GitCacheRestorationFailure(); + await materializeManagedProjectWorkspace(recordedBase, { repoUrl: checkout.source.repoUrl, resolveGitAuth: input.resolveGitAuth }); + if (!await isVerifiedNestedRepositoryBase(recordedBase, candidates!, checkout.source.repoUrl)) throw new GitCacheRestorationFailure(); + } + const base = await fs.realpath(recordedBase); + await withNestedRepositoryBaseLock(base, async () => { + let preserved: string[] = []; + if (checkout.restorePoint) { + const { payload, manifest } = await loadGitRestorePoint({ storage: input.storage ?? getStorageService(), workspace, ref: checkout.restorePoint }); + await importRestorePoint({ repositoryCwd: base, payload }); + preserved = manifest.preserved; + } + await ensureSavedGitCommit(base, checkout.source.repoUrl, checkout.commit, input.resolveGitAuth); + if (checkout.branchName) await gitCacheCommand(base, ["check-ref-format", "--branch", checkout.branchName]); + await restoreNestedRepositoryWorktree({ base, cwd: destination, commit: checkout.commit, branchName: checkout.branchName, + repoRef: checkout.source.repoRef, preserved, + afterAdd: async cwd => { + if (checkout.restorePoint) await applyGitRestorePointToWorktree({ storage: input.storage ?? getStorageService(), workspace, + ref: checkout.restorePoint, repositoryCwd: cwd, worktreePath: cwd }); + await restoreSettings(cwd, checkout.settings, input, relative); + await input.authorizeSource(checkout.source); + await recheck?.(); + } }); + }); + continue; + } const result = await materializeManagedProjectWorkspace(destination, { repoUrl: checkout.source.repoUrl, resolveGitAuth: input.resolveGitAuth, beforePublish: async temporary => { if (checkout.restorePoint) await ensureGitRestorePointCommit({ storage: input.storage ?? getStorageService(), workspace, diff --git a/server/src/services/git-workspace-expiration.ts b/server/src/services/git-workspace-expiration.ts index c8e4bf2cda4..516ca8d5fb5 100644 --- a/server/src/services/git-workspace-expiration.ts +++ b/server/src/services/git-workspace-expiration.ts @@ -12,6 +12,7 @@ import { currentRestorePointSnapshot, RestorePointSkip, type RestorePointLimits import { abandonGitRestorePoint, abandonGitRestorePointsByKey, activateGitRestorePoints, restorePointHistoryUnchanged, restorePointSnapshotMatches, storeGitRestorePoint } from "./git-restore-point-storage.js"; import { withIsolatedRepositorySourceLock, verifiedRecordedIsolatedRemoteSource } from "./isolated-remote-workspace-source.js"; import { managedProjectCheckoutName } from "./managed-project-checkout-name.js"; +import { nestedRepositoryBaseCandidates } from "./nested-repository-worktrees.js"; import type { GitRemoteAuthProvider } from "./git-credentials.js"; import { assertReproducibleGitFiles, assertReproducibleBaseGitStorage, assertReproducibleManagedCheckout, provenManagedCheckoutHistory, inspectGitCacheRecipe, inspectGitCacheRecipeForRestorePoint, readGitCacheRestoration, GIT_CACHE_RESTORATION_SCHEMA_V2, verifyRemoteGitCommit, gitCacheExpirationLockLocations, recoverGitCacheExpirationLocks, GitCacheRestorationFailure, type ManagedCheckoutProver, gitCacheCommand, readGitCacheExpiration, finishGitCacheTrash, type GitCacheExpiration, type GitCacheRestoration, GIT_CACHE_RESTORATION_KEY, GIT_CACHE_EXPIRATION_REASON } from "./git-workspace-restoration.js"; @@ -236,10 +237,11 @@ function managedCheckoutProver(db: Db, workspace: Workspace, auth: GitRemoteAuth const source = selectRestoredProjectRepositories(await rows, readGitCacheRestoration(workspace)?.contents).find(row => row.repoUrl && managedProjectCheckoutName({ id: row.id, repoUrl: row.repoUrl, repoRef: row.repoRef, cwd: row.cwd }) === name); if (!source?.repoUrl) throw new Error("unknown_managed_checkout"); - try { await assertReproducibleManagedCheckout(checkoutPath, { repoUrl: source.repoUrl, resolveGitAuth: auth, signal }); } + const baseCandidates = nestedRepositoryBaseCandidates({ companyId: workspace.companyId, projectId: workspace.projectId, repoUrl: source.repoUrl }); + try { await assertReproducibleManagedCheckout(checkoutPath, { repoUrl: source.repoUrl, resolveGitAuth: auth, signal, baseCandidates }); } catch (error) { if (!restorePoints || !CHECKOUT_RESTORE_POINT_RELAXABLE.has(skipReason(error))) throw error; - const { headOnRemote } = await assertReproducibleManagedCheckout(checkoutPath, { repoUrl: source.repoUrl, resolveGitAuth: auth, signal, restorePoint: true }); + const { headOnRemote } = await assertReproducibleManagedCheckout(checkoutPath, { repoUrl: source.repoUrl, resolveGitAuth: auth, signal, restorePoint: true, baseCandidates }); restorePoints.set(name, { headOnRemote, path: checkoutPath, repoUrl: source.repoUrl }); } checkouts.set(name, await readManagedCheckoutContents(checkoutPath, name, { ...source, repoUrl: source.repoUrl }, signal)); @@ -366,7 +368,7 @@ export async function expireReconstructableGitWorkspace(db: Db, snapshot: Worksp await persist(); } await recoverGitCacheExpirationLocks(workspace!); - const lockLocations = gitCacheExpirationLockLocations(workspace!); + const lockLocations = await gitCacheExpirationLockLocations(workspace!); await options.beforeDelete?.(workspace!); if (!await isGitWorkspaceExpirationEligible(tx, workspace, options.now, options.idleCutoff)) throw new Error("live_or_unresolved_consumer"); if (!state.workspaceRemoved && present) { diff --git a/server/src/services/git-workspace-restoration.ts b/server/src/services/git-workspace-restoration.ts index 45eab45fd0a..85ba8036811 100644 --- a/server/src/services/git-workspace-restoration.ts +++ b/server/src/services/git-workspace-restoration.ts @@ -18,6 +18,7 @@ import { ensureGitRestorePointCommit, validateGitRestorePointRef, type GitRestor import { getStorageService } from "../storage/index.js"; import type { StorageService } from "../storage/types.js"; import { PROJECT_REPOSITORIES_DIR } from "@paperclipai/adapter-utils/git-workspace-sync"; +import { forgetExpiredNestedRepositoryWorktree, nestedRepositoryBaseCandidates, readLinkedNestedCheckout, recordedNestedRepositoryBase, type LinkedNestedCheckout } from "./nested-repository-worktrees.js"; const execFile = promisify(execFileCallback); const shaPattern = /^[a-f0-9]{40}$/; @@ -248,40 +249,84 @@ async function assertCommitMessageSaved(cwd: string, gitDir: string, signal?: Ab if (!saved.includes(message.trim())) throw new Error("uncommitted_git_message"); } +const OWNED_NESTED_REF_LIMIT = 64; + +async function ownedNestedRefs(linked: LinkedNestedCheckout, signal?: AbortSignal) { + const refs = (await gitCacheCommand(linked.checkout, ["for-each-ref", `--count=${OWNED_NESTED_REF_LIMIT + 1}`, + "--format=%(objectname) %(refname)", `refs/heads/${linked.branchPrefix}`], signal)) + .split("\n").map(line => line.trim().split(" ")).filter(([sha, ref]) => shaPattern.test(sha ?? "") && ref) as Array<[string, string]>; + if (refs.length > OWNED_NESTED_REF_LIMIT) throw new Error("managed_checkout_local_tip_budget"); + return refs; +} + +async function ownedNestedReflogs(linked: LinkedNestedCheckout, refs: Array<[string, string]>, signal?: AbortSignal) { + const commits = new Set(); + for (const [, ref] of refs) { + for (const line of (await gitCacheCommand(linked.checkout, ["reflog", "show", "--format=%H", ref], signal).catch(() => "")).split("\n")) { + if (shaPattern.test(line.trim())) commits.add(line.trim()); + } + if (commits.size > MANAGED_CHECKOUT_LOCAL_TIP_LIMIT) throw new Error("managed_checkout_local_tip_budget"); + } + return [...commits]; +} + +async function assertOwnedNestedHead(linked: LinkedNestedCheckout, signal?: AbortSignal) { + const head = (await gitCacheCommand(linked.checkout, ["symbolic-ref", "--quiet", "HEAD"], signal).catch(() => "")).trim(); + if (head && !head.startsWith(`refs/heads/${linked.branchPrefix}`)) throw new Error("unowned_nested_worktree_branch"); +} + export async function assertReproducibleManagedCheckout(checkout: string, input: { - repoUrl: string; resolveGitAuth: GitRemoteAuthProvider; signal?: AbortSignal; restorePoint?: boolean; + repoUrl: string; resolveGitAuth: GitRemoteAuthProvider; signal?: AbortSignal; restorePoint?: boolean; baseCandidates?: string[]; }): Promise<{ headOnRemote: boolean; preserve: string[] }> { const { signal } = input; - const gitDir = await fs.lstat(path.join(checkout, ".git")).catch(() => null); - if (!gitDir?.isDirectory()) throw new Error("unproven_managed_checkout"); - if ((await gitCacheCommand(checkout, ["rev-parse", "--show-toplevel"], signal)).trim() !== await fs.realpath(checkout)) { - throw new Error("unproven_managed_checkout"); + const linked = await readLinkedNestedCheckout(checkout, input.baseCandidates ?? []); + if (linked) { + if ((await gitCacheCommand(checkout, ["config", "--local", "--null", "--get-all", "remote.origin.url"], signal)) !== `${input.repoUrl}\0`) { + throw new Error("unproven_managed_checkout"); + } + await assertOwnedNestedHead(linked, signal); + } else { + const gitDir = await fs.lstat(path.join(checkout, ".git")).catch(() => null); + if (!gitDir?.isDirectory()) throw new Error("unproven_managed_checkout"); + if ((await gitCacheCommand(checkout, ["rev-parse", "--show-toplevel"], signal)).trim() !== await fs.realpath(checkout)) { + throw new Error("unproven_managed_checkout"); + } + const worktrees = (await gitCacheCommand(checkout, ["worktree", "list", "--porcelain"], signal)) + .split("\n").filter(line => line.startsWith("worktree ")); + if (worktrees.length !== 1) throw new Error("managed_checkout_has_worktrees"); + } + await assertReproducibleGitFiles(checkout, signal, { preserveSettings: true, restorePoint: input.restorePoint, + ...(linked ? { verifyFetchedCommit: (commit: string, verificationSignal?: AbortSignal) => + verifyRemoteGitCommit(input.repoUrl, commit, input.resolveGitAuth, verificationSignal) } : {}) }); + if (!linked) { + await assertReproducibleBaseGitStorage(checkout, signal, { savedCommitMessage: true }); + await assertCommitMessageSaved(checkout, path.join(checkout, ".git"), signal); } - const worktrees = (await gitCacheCommand(checkout, ["worktree", "list", "--porcelain"], signal)) - .split("\n").filter(line => line.startsWith("worktree ")); - if (worktrees.length !== 1) throw new Error("managed_checkout_has_worktrees"); - await assertReproducibleGitFiles(checkout, signal, { preserveSettings: true, restorePoint: input.restorePoint }); - await assertReproducibleBaseGitStorage(checkout, signal, { savedCommitMessage: true }); - await assertCommitMessageSaved(checkout, path.join(checkout, ".git"), signal); const refs = (await gitCacheCommand(checkout, ["for-each-ref", "--format=%(refname)"], signal)).trim().split("\n").filter(Boolean); if (refs.some(ref => !/^refs\/(heads|tags|remotes\/origin)\//.test(ref))) throw new Error("unsupported_managed_checkout_refs"); if (input.restorePoint) { const { headOnRemote, preserve } = await provenManagedCheckoutHistory(checkout, input.repoUrl, input.resolveGitAuth, signal); - const unreachable = await gitCacheCommand(checkout, ["fsck", "--connectivity-only", "--unreachable", "--no-progress"], signal); - if (unreachable.trim()) throw new Error("unreachable_managed_checkout_objects"); + if (!linked) { + const unreachable = await gitCacheCommand(checkout, ["fsck", "--connectivity-only", "--unreachable", "--no-progress"], signal); + if (unreachable.trim()) throw new Error("unreachable_managed_checkout_objects"); + } return { headOnRemote, preserve }; } + const owned = linked ? await ownedNestedRefs(linked, signal) : []; const localTips = new Set([ ...(await gitCacheCommand(checkout, ["rev-parse", "--verify", "HEAD^{commit}"], signal)).split("\n"), - ...(await gitCacheCommand(checkout, ["for-each-ref", "--format=%(objectname)"], signal)).split("\n"), - ...(await gitCacheCommand(checkout, ["reflog", "show", "--all", "--format=%H"], signal).catch(() => "")).split("\n"), + ...linked ? owned.map(([sha]) => sha) : (await gitCacheCommand(checkout, ["for-each-ref", "--format=%(objectname)"], signal)).split("\n"), + ...(await gitCacheCommand(checkout, ["reflog", "show", ...linked ? [] : ["--all"], "--format=%H", ...linked ? ["HEAD"] : []], signal).catch(() => "")).split("\n"), + ...linked ? await ownedNestedReflogs(linked, owned, signal) : [], ].map(line => line.trim()).filter(sha => shaPattern.test(sha))); if (localTips.size > MANAGED_CHECKOUT_LOCAL_TIP_LIMIT) throw new Error("managed_checkout_local_tip_budget"); if (localTips.size && await hasCommitsMissingUpstream(checkout, [...localTips], input.repoUrl, input.resolveGitAuth, signal)) { throw new Error("unpushed_managed_checkout"); } - const unreachable = await gitCacheCommand(checkout, ["fsck", "--connectivity-only", "--unreachable", "--no-progress"], signal); - if (unreachable.trim()) throw new Error("unreachable_managed_checkout_objects"); + if (!linked) { + const unreachable = await gitCacheCommand(checkout, ["fsck", "--connectivity-only", "--unreachable", "--no-progress"], signal); + if (unreachable.trim()) throw new Error("unreachable_managed_checkout_objects"); + } return { headOnRemote: true, preserve: [] }; } @@ -289,6 +334,8 @@ const MANAGED_CHECKOUT_LOCAL_TIP_LIMIT = 5_000; export async function provenManagedCheckoutHistory(checkout: string, repoUrl: string, resolveGitAuth: GitRemoteAuthProvider, signal?: AbortSignal): Promise<{ headOnRemote: boolean; preserve: string[] }> { + const linked = await readLinkedNestedCheckout(checkout); + if (linked) return provenNestedWorktreeHistory(linked, repoUrl, resolveGitAuth, signal); const head = (await gitCacheCommand(checkout, ["rev-parse", "--verify", "HEAD^{commit}"], signal)).trim(); const shas = (output: string) => output.split("\n").map(line => line.trim()).filter(sha => shaPattern.test(sha)); const covered = new Set([head, @@ -324,6 +371,31 @@ export async function provenManagedCheckoutHistory(checkout: string, repoUrl: st return { headOnRemote, preserve }; } +async function provenNestedWorktreeHistory(linked: LinkedNestedCheckout, repoUrl: string, resolveGitAuth: GitRemoteAuthProvider, + signal?: AbortSignal): Promise<{ headOnRemote: boolean; preserve: string[] }> { + const checkout = linked.checkout; + await assertOwnedNestedHead(linked, signal); + const head = (await gitCacheCommand(checkout, ["rev-parse", "--verify", "HEAD^{commit}"], signal)).trim(); + const shas = (output: string) => output.split("\n").map(line => line.trim()).filter(sha => shaPattern.test(sha)); + const covered = new Set([head, + ...shas(await gitCacheCommand(checkout, ["reflog", "show", "--format=%H", "HEAD"], signal).catch(() => "")), + ...shas(await gitCacheCommand(checkout, ["rev-parse", "--verify", "-q", "ORIG_HEAD^{commit}"], signal).catch(() => ""))]); + const owned = await ownedNestedRefs(linked, signal); + const recorded = new Set([...owned.map(([sha]) => sha), ...await ownedNestedReflogs(linked, owned, signal)]); + if (recorded.size + covered.size > MANAGED_CHECKOUT_LOCAL_TIP_LIMIT) throw new Error("managed_checkout_local_tip_budget"); + const current = (await gitCacheCommand(checkout, ["symbolic-ref", "--quiet", "HEAD"], signal).catch(() => "")).trim(); + const otherTips = owned.filter(([, ref]) => ref !== current).map(([sha]) => sha); + if (otherTips.length && await hasCommitsMissingUpstream(checkout, otherTips, repoUrl, resolveGitAuth, signal, [head])) { + throw new Error("unpushed_managed_checkout_branches"); + } + const uncovered = [...recorded].filter(sha => !covered.has(sha)); + if (uncovered.length && await hasCommitsMissingUpstream(checkout, uncovered, repoUrl, resolveGitAuth, signal, [...covered])) { + throw new Error("unpushed_managed_checkout_branches"); + } + const headOnRemote = !await hasCommitsMissingUpstream(checkout, [head], repoUrl, resolveGitAuth, signal); + return { headOnRemote, preserve: [] }; +} + async function tipsMissingUpstream(checkout: string, tips: string[], repoUrl: string, resolveGitAuth: GitRemoteAuthProvider, signal?: AbortSignal, covered: string[] = []) { const missing = new Set(await commitsMissingUpstream(checkout, tips, repoUrl, resolveGitAuth, signal, covered)); @@ -342,7 +414,8 @@ async function commitsMissingUpstream(checkout: string, localTips: string[], rep const options = { timeout: 120_000, signal, maxBuffer: 16 * 1024 * 1024, cwd: os.tmpdir(), env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: os.devNull } }; await execFile("git", ["init", "--bare", "--quiet", store], options); - await fs.writeFile(path.join(store, "objects", "info", "alternates"), `${path.join(await fs.realpath(checkout), ".git", "objects")}\n`); + const common = (await gitCacheCommand(checkout, ["rev-parse", "--path-format=absolute", "--git-common-dir"], signal)).trim(); + await fs.writeFile(path.join(store, "objects", "info", "alternates"), `${path.join(await fs.realpath(common), "objects")}\n`); try { const auth = await resolveGitAuth(repoUrl); await execFile("git", [...auth?.configArgs ?? [], "-C", store, "fetch", "--quiet", "--no-tags", "--no-recurse-submodules", @@ -416,7 +489,7 @@ export async function assertReproducibleGitFiles(cwd: string, signal?: AbortSign if (!options.restorePoint && (await gitCacheCommand(cwd, ["status", "--porcelain=v1", "--untracked-files=all", "-z"], signal)).length) throw new Error("dirty_or_untracked_files"); if ((await gitCacheCommand(cwd, ["ls-files", "-v"], signal)).split("\n").some(line => /^[a-zS] /.test(line))) throw new Error("unsupported_index_flags"); const localConfig = await gitCacheCommand(cwd, ["config", "--local", "--name-only", "--list"], signal); - if (localConfig.split("\n").filter(Boolean).some(key => !/^(core\.(repositoryformatversion|filemode|bare|logallrefupdates|ignorecase|precomposeunicode)|remote\.origin\.(url|fetch)|branch\.[^.]+\.(remote|merge))$/.test(key))) throw new Error("unsupported_local_git_config"); + if (localConfig.split("\n").filter(Boolean).some(key => !/^(core\.(repositoryformatversion|filemode|bare|logallrefupdates|ignorecase|precomposeunicode)|remote\.origin\.(url|fetch)|branch\..+\.(remote|merge))$/.test(key))) throw new Error("unsupported_local_git_config"); const regenerable = await findRegenerableWorkspaceDirs(cwd, { signal }); const ignored = (await gitCacheCommand(cwd, ["ls-files", "--others", "--ignored", "--exclude-standard", "-z"], signal)).split("\0").filter(Boolean); const unexplained = ignored.filter(file => !(options.preserveSettings && file === PAPERCLIP_CLAUDE_SETTINGS_FILE) && !regenerable.some(dir => file.startsWith(`${dir}/`)) @@ -541,7 +614,7 @@ export function readGitCacheExpiration(workspace: Workspace): GitCacheExpiration return state; } -export function gitCacheExpirationLockLocations(workspace: Workspace): GitExpirationLockLocation[] { +export async function gitCacheExpirationLockLocations(workspace: Workspace): Promise { const state = readGitCacheExpiration(workspace); const recipe = readGitCacheRestoration(workspace); if (!state || !recipe || !state.lockToken || state.completed) return []; @@ -555,6 +628,14 @@ export function gitCacheExpirationLockLocations(workspace: Workspace): GitExpira { root: recipe.sourcePath, relativePath: path.join(".git", branchPath(recipe.branchName)), checkout: null }, ]; for (const checkout of recipe.contents?.checkouts ?? []) { + if (checkout.worktree) { + const base = await recordedNestedBase(recipe, checkout.source.repoUrl, checkout.worktree.base); + for (const name of ["index.lock", "HEAD.lock"]) { + locations.push({ root: base, relativePath: path.join(".git", "worktrees", checkout.worktree.admin, name), checkout: checkout.name }); + } + if (checkout.branchName) locations.push({ root: base, relativePath: path.join(".git", branchPath(checkout.branchName)), checkout: checkout.name }); + continue; + } for (const name of ["index.lock", "HEAD.lock", ...(checkout.branchName ? [branchPath(checkout.branchName)] : [])]) { locations.push({ root: recipe.worktreePath, relativePath: path.join(PROJECT_REPOSITORIES_DIR, checkout.name, ".git", name), checkout: checkout.name }); } @@ -562,6 +643,14 @@ export function gitCacheExpirationLockLocations(workspace: Workspace): GitExpira return locations; } +async function recordedNestedBase(recipe: GitCacheRestoration, repoUrl: string, recorded: string) { + if (!recipe.projectId) throw new GitCacheRestorationFailure(); + const base = await recordedNestedRepositoryBase(recorded, + nestedRepositoryBaseCandidates({ companyId: recipe.companyId, projectId: recipe.projectId, repoUrl })); + if (!base) throw new GitCacheRestorationFailure(); + return base; +} + export async function recoverGitCacheExpirationLocks(workspace: Workspace) { const state = readGitCacheExpiration(workspace); if (!state?.lockToken || state.completed || !["archived", "cleanup_failed"].includes(workspace.status ?? "")) return; @@ -570,7 +659,7 @@ export async function recoverGitCacheExpirationLocks(workspace: Workspace) { if (error.code === "ENOENT") return false; throw error; })) await verifiedGitCacheSource(workspace, recipe.sourcePath); - await recoverGitExpirationLocks(gitCacheExpirationLockLocations(workspace), state.lockToken); + await recoverGitExpirationLocks(await gitCacheExpirationLockLocations(workspace), state.lockToken); } // Both restart recovery and reopening finish only recorded, uniquely named trash. @@ -601,6 +690,15 @@ export async function finishGitCacheTrash(workspace: Workspace) { if (pointer !== expected) throw new GitCacheRestorationFailure(); await fs.rm(state.worktreeAdmin, { recursive: true, force: true }); } + if (await exists(recipe.worktreePath)) return; + for (const checkout of recipe.contents?.checkouts ?? []) { + if (!checkout.worktree) continue; + const base = await recordedNestedBase(recipe, checkout.source.repoUrl, checkout.worktree.base); + if (!await exists(base)) continue; + await forgetExpiredNestedRepositoryWorktree({ base: await fs.realpath(base), admin: checkout.worktree.admin, + checkout: path.join(await fs.realpath(path.dirname(recipe.worktreePath)), path.basename(recipe.worktreePath), PROJECT_REPOSITORIES_DIR, checkout.name), + branchName: checkout.branchName, commit: checkout.commit }); + } } /** Called only after authorization and while holding the source/lifecycle fences. */ @@ -649,7 +747,7 @@ export async function ensureSavedGitCommit(cwd: string, repoUrl: string, commit: if (await gitCacheCommand(cwd, ["cat-file", "-e", `${commit}^{commit}`]).then(() => true, () => false)) return; try { const auth = await resolveGitAuth(repoUrl); - await execFile("git", [...auth?.configArgs ?? [], "-C", cwd, "fetch", "--no-tags", "--no-recurse-submodules", "--", repoUrl, commit], { + await execFile("git", [...auth?.configArgs ?? [], "-c", `core.hooksPath=${os.devNull}`, "-C", cwd, "fetch", "--no-tags", "--no-recurse-submodules", "--", repoUrl, commit], { timeout: 60_000, maxBuffer: 1024 * 1024, env: { ...process.env, ...auth?.env, GIT_TERMINAL_PROMPT: "0" }, }); if ((await gitCacheCommand(cwd, ["rev-parse", `${commit}^{commit}`])).trim() !== commit) throw new GitCacheRestorationFailure(); diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts index 156555e71a8..0b40ec91564 100644 --- a/server/src/services/heartbeat/workspace-preparation.test.ts +++ b/server/src/services/heartbeat/workspace-preparation.test.ts @@ -640,10 +640,7 @@ describe.skipIf(!support.supported)("heartbeat workspace preparation boundary", const namespace = join(source, ".paperclip-repositories"); const guard = vi.spyOn(input.services.executionWorkspacesSvc, "assertTaskRepositoryPublicationAllowed") .mockImplementation(async () => { - const candidates = await readdir(namespace); - expect(candidates).toHaveLength(1); - expect(candidates[0]).toContain(".clone-"); - expect(await readFile(join(namespace, candidates[0]!, "README.md"), "utf8")).toBe("source content\n"); + expect(await readdir(namespace)).toEqual([]); throw new Error("Task binding changed during clone"); }); await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("Task binding changed during clone"); diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts index 68ce0e2db5b..c083c51f263 100644 --- a/server/src/services/heartbeat/workspace-preparation.ts +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -1050,6 +1050,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace anchorRepoUrl: executionWorkspace.repoUrl, workspaces: repositoryRows, restoredContents: persistedExecutionWorkspace ? readGitCacheRestoration(persistedExecutionWorkspace)?.contents : undefined, + sharedBase: { companyId: agent.companyId, projectId: executionWorkspace.projectId! }, resolveGitAuth: workspaceGitAuthProvider, prepareLocalSource: persistedExecutionWorkspace && issueId ? (sourceCwd, projectWorkspaceId) => executionWorkspacesSvc.prepareConfiguredRepositoryCopy({ diff --git a/server/src/services/heartbeat/workspaces.ts b/server/src/services/heartbeat/workspaces.ts index 5e4ef54f870..c607e2426e3 100644 --- a/server/src/services/heartbeat/workspaces.ts +++ b/server/src/services/heartbeat/workspaces.ts @@ -7,6 +7,7 @@ import { deriveRepoNameFromRepoUrl, managedProjectCheckoutName, projectRepositor import { forbidden } from "../../errors.js"; import { readGitConnectionFailure } from "../git-connection-failure.js"; import { materializeManagedProjectWorkspace, ensureManagedRepositoriesIgnored } from "../managed-repository-checkout.js"; +import { addNestedRepositoryWorktree, isVerifiedNestedRepositoryBase, nestedRepositoryBaseCandidates, readLinkedNestedCheckout, repairMovedNestedRepositoryWorktree } from "../nested-repository-worktrees.js"; export { materializeManagedProjectWorkspace } from "../managed-repository-checkout.js"; import { sql } from "drizzle-orm"; import { materializeIsolatedTaskDirectory } from "../isolated-task-directory.js"; @@ -503,6 +504,7 @@ export async function prepareProjectRepositoryWorkspaces(input: { beforePublish?: () => Promise; prepareLocalSource?: (sourceCwd: string, projectWorkspaceId: string) => Promise<() => Promise>; restoredContents?: GitWorkspaceContents; + sharedBase?: { companyId: string; projectId: string }; }): Promise> { const identity = projectRepositoryIdentity; const seen = new Set(input.anchorRepoUrl ? [identity(input.anchorRepoUrl)] : []); @@ -527,6 +529,11 @@ export async function prepareProjectRepositoryWorkspaces(input: { const existing = await fs.lstat(cwd).catch(() => null); if (!existing && input.restoredContents?.checkouts.some(checkout => checkout.name === key)) throw new GitCacheRestorationFailure(); if (existing && (!existing.isDirectory() || existing.isSymbolicLink())) throw new Error("Invalid project repository checkout path"); + const baseCandidates = input.sharedBase ? nestedRepositoryBaseCandidates({ ...input.sharedBase, repoUrl }) : null; + const linked = existing ? await readLinkedNestedCheckout(cwd, baseCandidates ?? []).catch(() => { + throw new Error("Invalid project repository checkout path"); + }) : null; + if (linked && !await isVerifiedNestedRepositoryBase(linked.base, baseCandidates!, repoUrl)) throw new Error("Invalid project repository checkout path"); const configuredLocalSource = workspace.cwd && workspace.cwd !== REPO_ONLY_CWD_SENTINEL ? workspace.cwd : null; const localSource = configuredLocalSource && await fs.stat(configuredLocalSource).then((entry) => entry.isDirectory()).catch(() => false) @@ -539,8 +546,17 @@ export async function prepareProjectRepositoryWorkspaces(input: { await recheckLocalSource?.(); await input.beforePublish?.(); }; - const result = await materializeManagedProjectWorkspace(cwd, { repoUrl, repoRef: workspace.repoRef, localSource, resolveGitAuth: input.resolveGitAuth, beforePublish }); - if (result.warning) throw new Error(result.warning); + if (!linked) { + const shared = input.sharedBase && !existing && !localSource + ? await ensureManagedProjectWorkspace({ ...input.sharedBase, repoUrl, resolveGitAuth: input.resolveGitAuth }) : null; + if (shared && !shared.warning && await isVerifiedNestedRepositoryBase(shared.cwd, baseCandidates!, repoUrl)) { + await addNestedRepositoryWorktree({ base: shared.cwd, cwd, repoUrl, repoRef: workspace.repoRef, + resolveGitAuth: input.resolveGitAuth, beforeAdd: beforePublish }); + } else { + const result = await materializeManagedProjectWorkspace(cwd, { repoUrl, repoRef: workspace.repoRef, localSource, resolveGitAuth: input.resolveGitAuth, beforePublish }); + if (result.warning) throw new Error(result.warning); + } + } await beforePublish(); results.push({ workspaceId: workspace.id, cwd, repoUrl, repoRef: workspace.repoRef }); } @@ -549,8 +565,10 @@ export async function prepareProjectRepositoryWorkspaces(input: { for (const entry of await fs.readdir(root)) { if (active.has(entry) || entry.startsWith("task-repo-") || entry.includes(".clone-")) continue; const retained = path.join(input.cwd, ".paperclip-runtime", "detached-repositories", randomUUID()); + const linked = await readLinkedNestedCheckout(path.join(root, entry)).catch(() => null); await fs.mkdir(path.dirname(retained), { recursive: true }); await fs.rename(path.join(root, entry), retained); + if (linked) await repairMovedNestedRepositoryWorktree(linked.base, retained); } return results; } diff --git a/server/src/services/nested-repository-worktrees.ts b/server/src/services/nested-repository-worktrees.ts new file mode 100644 index 00000000000..ae48ab2c947 --- /dev/null +++ b/server/src/services/nested-repository-worktrees.ts @@ -0,0 +1,333 @@ +import fs from "node:fs/promises"; +import { constants } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { createHash } from "node:crypto"; +import { AsyncLocalStorage } from "node:async_hooks"; +import { execFile as execFileCallback } from "node:child_process"; +import { promisify } from "node:util"; +import { PROJECT_REPOSITORIES_DIR } from "@paperclipai/adapter-utils/git-workspace-sync"; +import { resolveManagedProjectWorkspaceDir } from "../home-paths.js"; +import { deriveRepoNameFromRepoUrl } from "./managed-project-checkout-name.js"; +import { scrubGitCredentialText, type GitRemoteAuthProvider } from "./git-credentials.js"; +import { sanitizeRuntimeServiceBaseEnv } from "./runtime-service-env.js"; + +const execFile = promisify(execFileCallback); +const NESTED_WORKTREE_GIT_TIMEOUT_MS = 10 * 60 * 1000; +const NESTED_WORKTREE_BRANCH = /^paperclip\/[a-f0-9]{12}\/./; +export const DETACHED_REPOSITORIES_DIR = path.join(".paperclip-runtime", "detached-repositories"); + +export type LinkedNestedCheckout = { checkout: string; base: string; gitDir: string; admin: string; branchPrefix: string }; +type GitInvocation = { configArgs?: string[]; env?: Record; timeout?: number }; + +async function git(cwd: string, args: string[], options: GitInvocation = {}) { + const result = await execFile("git", [...options.configArgs ?? [], "-c", "core.fsmonitor=false", "-c", `core.hooksPath=${os.devNull}`, "-C", cwd, ...args], { + timeout: options.timeout ?? 60_000, maxBuffer: 16 * 1024 * 1024, + env: { ...sanitizeRuntimeServiceBaseEnv(process.env), GIT_TERMINAL_PROMPT: "0", ...options.env }, + }); + return result.stdout; +} + +async function missing(target: string) { + return fs.lstat(target).then(() => false, (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return true; + throw error; + }); +} + +async function canonicalPath(target: string) { + return path.join(await fs.realpath(path.dirname(target)), path.basename(target)); +} + +const WORKTREE_POINTER_LIMIT = 4096; + +export async function readWorktreePointer(admin: string) { + const target = path.join(admin, "gitdir"); + const entry = await fs.lstat(target); + if (!entry.isFile() || entry.nlink !== 1 || entry.size > WORKTREE_POINTER_LIMIT) throw new Error("unproven_managed_checkout"); + const handle = await fs.open(target, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = await handle.stat(); + if (!stat.isFile() || stat.nlink !== 1 || stat.size > WORKTREE_POINTER_LIMIT) throw new Error("unproven_managed_checkout"); + const buffer = Buffer.alloc(WORKTREE_POINTER_LIMIT + 1); + let length = 0; + while (length < buffer.length) { + const { bytesRead } = await handle.read(buffer, length, buffer.length - length, length); + if (bytesRead === 0) break; + length += bytesRead; + } + if (length > WORKTREE_POINTER_LIMIT) throw new Error("unproven_managed_checkout"); + return buffer.subarray(0, length).toString("utf8").trim(); + } finally { await handle.close(); } +} + +export function nestedRepositoryBaseCandidates(input: { companyId: string; projectId: string; repoUrl: string }) { + const root = resolveManagedProjectWorkspaceDir({ companyId: input.companyId, projectId: input.projectId, + repoName: deriveRepoNameFromRepoUrl(input.repoUrl) }); + return [root, `${root}-${createHash("sha256").update(input.repoUrl).digest("hex").slice(0, 12)}`]; +} + +async function looseCanonicalPath(target: string): Promise { + const real = await fs.realpath(target).catch(() => null); + if (real) return real; + const parent = path.dirname(target); + return parent === target ? target : path.join(await looseCanonicalPath(parent), path.basename(target)); +} + +export async function recordedNestedRepositoryBase(recorded: string, candidates: string[]) { + for (const candidate of candidates) if (await looseCanonicalPath(candidate) === recorded) return candidate; + return null; +} + +export async function nestedWorktreeBranchPrefix(checkout: string) { + const id = createHash("sha256").update(await canonicalPath(checkout)).digest("hex").slice(0, 12); + return `paperclip/${id}/`; +} + +export function isNestedWorktreeBranchName(name: string) { + return NESTED_WORKTREE_BRANCH.test(name); +} + +export async function isVerifiedNestedRepositoryBase(base: string, candidates: string[], repoUrl: string) { + const real = await fs.realpath(base).catch(() => null); + if (!real) return false; + const allowed = await Promise.all(candidates.map(candidate => fs.realpath(candidate).catch(() => null))); + if (!allowed.includes(real)) return false; + const storage = await fs.lstat(path.join(real, ".git")).catch(() => null); + if (!storage?.isDirectory() || storage.isSymbolicLink()) return false; + const common = (await git(real, ["rev-parse", "--path-format=absolute", "--git-common-dir"]).catch(() => "")).trim(); + if (common !== path.join(real, ".git")) return false; + return (await git(real, ["config", "--local", "--null", "--get-all", "remote.origin.url"]).catch(() => "")) === `${repoUrl}\0`; +} + +export async function readLinkedNestedCheckout(checkout: string, baseCandidates?: string[]): Promise { + const marker = await fs.lstat(path.join(checkout, ".git")).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + if (!marker || marker.isDirectory()) return null; + if (!marker.isFile()) throw new Error("unproven_managed_checkout"); + const real = await canonicalPath(checkout); + if (await fs.realpath(checkout) !== real) throw new Error("unproven_managed_checkout"); + const gitDir = await fs.realpath((await git(real, ["rev-parse", "--absolute-git-dir"])).trim()); + const common = await fs.realpath((await git(real, ["rev-parse", "--path-format=absolute", "--git-common-dir"])).trim()); + if (path.basename(common) !== ".git" || path.dirname(gitDir) !== path.join(common, "worktrees")) throw new Error("unproven_managed_checkout"); + const pointer = await readWorktreePointer(gitDir); + if (path.basename(pointer) !== ".git" || path.join(await fs.realpath(path.dirname(pointer)).catch(() => ""), ".git") !== path.join(real, ".git")) { + throw new Error("unproven_managed_checkout"); + } + if ((await git(real, ["rev-parse", "--show-toplevel"])).trim() !== real) throw new Error("unproven_managed_checkout"); + const base = path.dirname(common); + const storage = await fs.lstat(common); + if (!storage.isDirectory() || storage.isSymbolicLink()) throw new Error("unproven_managed_checkout"); + if (baseCandidates) { + const allowed = await Promise.all(baseCandidates.map(candidate => fs.realpath(candidate).catch(() => null))); + if (!allowed.includes(base)) throw new Error("unproven_managed_checkout"); + } + return { checkout: real, base, gitDir, admin: path.basename(gitDir), branchPrefix: await nestedWorktreeBranchPrefix(real) }; +} + +const heldBaseLocks = new AsyncLocalStorage>(); +const baseLocks = new Map>(); + +export async function withNestedRepositoryBaseLock(base: string, action: () => Promise): Promise { + const key = await looseCanonicalPath(path.resolve(base)); + const held = heldBaseLocks.getStore() ?? new Set(); + if (held.has(key)) return action(); + const previous = baseLocks.get(key) ?? Promise.resolve(); + let release!: () => void; + const current = previous.then(() => new Promise(resolve => { release = resolve; })); + baseLocks.set(key, current); + await previous; + try { + return await heldBaseLocks.run(new Set([...held, key]), action); + } finally { + release(); + if (baseLocks.get(key) === current) baseLocks.delete(key); + } +} + +async function removeStaleRegistrations(base: string, checkout: string) { + const expected = path.join(checkout, ".git"); + const root = path.join(base, ".git", "worktrees"); + for (const admin of await fs.readdir(root).catch(() => [] as string[])) { + const directory = path.join(root, admin); + const entry = await fs.lstat(directory).catch(() => null); + if (!entry?.isDirectory() || entry.isSymbolicLink()) continue; + const pointer = await readWorktreePointer(directory).catch(() => ""); + if (pointer !== expected || !await missing(checkout)) continue; + await fs.rm(directory, { recursive: true, force: true }); + } +} + +async function checkedOutBranches(base: string) { + return new Set((await git(base, ["worktree", "list", "--porcelain"])).split("\n") + .filter(line => line.startsWith("branch ")).map(line => line.slice("branch ".length))); +} + +async function resolveStartPoint(base: string, repoRef: string | null) { + const commit = async (ref: string) => (await git(base, ["rev-parse", "--verify", "-q", `${ref}^{commit}`]).catch(() => "")).trim(); + const ref = repoRef?.trim().replace(/^refs\/heads\//, "") || null; + if (ref) { + if (await commit(`refs/remotes/origin/${ref}`)) return { branch: ref, start: `refs/remotes/origin/${ref}` }; + const detached = await commit(ref); + if (!detached) throw new Error(`Repository ref "${ref}" was not found`); + return { branch: null, start: detached }; + } + const remoteHead = (await git(base, ["symbolic-ref", "--quiet", "refs/remotes/origin/HEAD"]).catch(() => "")).trim(); + const fallback = (await git(base, ["symbolic-ref", "--quiet", "--short", "HEAD"]).catch(() => "")).trim(); + const branch = remoteHead.startsWith("refs/remotes/origin/") ? remoteHead.slice("refs/remotes/origin/".length) : fallback; + if (branch && await commit(`refs/remotes/origin/${branch}`)) return { branch, start: `refs/remotes/origin/${branch}` }; + return { branch: null, start: await commit("HEAD") }; +} + +class NestedWorktreeAuthorizationFailure extends Error { + constructor(readonly cause: unknown) { super("nested_worktree_authorization_failed"); } +} + +export async function addNestedRepositoryWorktree(input: { + base: string; + cwd: string; + repoUrl: string; + repoRef: string | null; + resolveGitAuth?: GitRemoteAuthProvider | null; + beforeAdd?: () => Promise; +}): Promise<{ branchName: string | null }> { + try { + return await withNestedRepositoryBaseLock(input.base, async () => { + const auth = input.resolveGitAuth ? await input.resolveGitAuth(input.repoUrl) : null; + await git(input.base, ["fetch", "--quiet", "--prune", "--no-write-fetch-head", "origin"], { configArgs: auth?.configArgs, + env: auth?.env, timeout: NESTED_WORKTREE_GIT_TIMEOUT_MS }); + if (!input.repoRef?.trim()) { + await git(input.base, ["remote", "set-head", "origin", "--auto"], { configArgs: auth?.configArgs, env: auth?.env }).catch(() => undefined); + } + const { branch, start } = await resolveStartPoint(input.base, input.repoRef); + if (!start) throw new Error("Repository has no commit to check out"); + const cwd = await canonicalPath(input.cwd); + await removeStaleRegistrations(input.base, cwd); + await input.beforeAdd?.().catch((error: unknown) => { throw new NestedWorktreeAuthorizationFailure(error); }); + if (!branch) { + await git(input.base, ["worktree", "add", "--quiet", "--detach", cwd, start], { timeout: NESTED_WORKTREE_GIT_TIMEOUT_MS }); + return { branchName: null }; + } + const prefix = await nestedWorktreeBranchPrefix(cwd); + const held = await checkedOutBranches(input.base); + let owned = `${prefix}${branch}`; + for (let attempt = 2; held.has(`refs/heads/${owned}`); attempt += 1) owned = `${prefix}r${attempt}/${branch}`; + await git(input.base, ["check-ref-format", "--branch", owned]); + const existing = (await git(input.base, ["rev-parse", "--verify", "-q", `refs/heads/${owned}`]).catch(() => "")).trim(); + await git(input.base, existing ? ["worktree", "add", "--quiet", cwd, owned] + : ["worktree", "add", "--quiet", "--track", "-b", owned, cwd, start], + { timeout: NESTED_WORKTREE_GIT_TIMEOUT_MS }); + return { branchName: owned }; + }); + } catch (error) { + if (error instanceof NestedWorktreeAuthorizationFailure) throw error.cause; + const reason = error instanceof Error ? error.message : String(error); + throw new Error(scrubGitCredentialText(`Failed to prepare managed worktree for "${input.repoUrl}" at "${input.cwd}": ${reason}`)); + } +} + +export async function restoreNestedRepositoryWorktree(input: { + base: string; + cwd: string; + commit: string; + branchName: string | null; + repoRef: string | null; + preserved?: readonly string[]; + afterAdd: (cwd: string) => Promise; +}) { + await withNestedRepositoryBaseLock(input.base, async () => { + const cwd = await canonicalPath(input.cwd); + const prefix = await nestedWorktreeBranchPrefix(cwd); + if (input.branchName && !input.branchName.startsWith(prefix)) throw new Error("unowned_nested_worktree_branch"); + await removeStaleRegistrations(input.base, cwd); + const existing = input.branchName + ? (await git(input.base, ["rev-parse", "--verify", "-q", `refs/heads/${input.branchName}`]).catch(() => "")).trim() : ""; + if (existing && existing !== input.commit) throw new Error("nested_worktree_branch_moved"); + await git(input.base, ["worktree", "add", "--quiet", "--detach", cwd, input.commit], { timeout: NESTED_WORKTREE_GIT_TIMEOUT_MS }); + try { + for (const commit of input.preserved ?? []) { + await git(cwd, ["update-ref", "--no-deref", "-m", "paperclip: restore preserved history", "HEAD", commit]); + } + if (input.preserved?.length) await git(cwd, ["update-ref", "--no-deref", "-m", "paperclip: restore saved commit", "HEAD", input.commit]); + if (input.branchName) { + await git(cwd, existing ? ["checkout", "--quiet", input.branchName] : ["checkout", "--quiet", "-b", input.branchName]); + const rest = input.branchName.slice(prefix.length); + const { branch: upstream } = await resolveStartPoint(input.base, input.repoRef).catch(() => ({ branch: null })); + if (upstream && (rest === upstream || rest.replace(/^r\d+\//, "") === upstream)) { + await git(cwd, ["branch", "--quiet", `--set-upstream-to=origin/${upstream}`, input.branchName]); + } + } + await input.afterAdd(cwd); + } catch (error) { + await git(input.base, ["worktree", "remove", "--force", cwd]).catch(() => undefined); + await removeStaleRegistrations(input.base, cwd).catch(() => undefined); + if (input.branchName && !existing) { + await git(input.base, ["update-ref", "-d", `refs/heads/${input.branchName}`, input.commit]).catch(() => undefined); + } + throw error; + } + }); +} + +export async function forgetExpiredNestedRepositoryWorktree(input: { + base: string; checkout: string; admin: string; branchName: string | null; commit: string; +}) { + await withNestedRepositoryBaseLock(input.base, async () => { + const admin = path.join(input.base, ".git", "worktrees", input.admin); + const entry = await fs.lstat(admin).catch(() => null); + if (entry && (!entry.isDirectory() || entry.isSymbolicLink())) throw new Error("unproven_worktree_admin"); + if (entry && await missing(input.checkout) + && await readWorktreePointer(admin).catch(() => "") === path.join(input.checkout, ".git")) { + await fs.rm(admin, { recursive: true, force: true }); + } + if (input.branchName && await missing(input.checkout)) { + await git(input.base, ["update-ref", "-d", `refs/heads/${input.branchName}`, input.commit]).catch(() => undefined); + } + }); +} + +export async function collectNestedRepositoryWorktrees(taskCwd: string): Promise { + const found: LinkedNestedCheckout[] = []; + for (const parent of [PROJECT_REPOSITORIES_DIR, DETACHED_REPOSITORIES_DIR]) { + const root = path.join(taskCwd, parent); + const rootEntry = await fs.lstat(root).catch(() => null); + if (!rootEntry?.isDirectory() || rootEntry.isSymbolicLink()) continue; + for (const name of await fs.readdir(root)) { + const linked = await readLinkedNestedCheckout(path.join(root, name)).catch(() => null); + if (!linked) continue; + const head = (await git(linked.checkout, ["symbolic-ref", "--quiet", "--short", "HEAD"]).catch(() => "")).trim(); + found.push(isNestedWorktreeBranchName(head) ? { ...linked, branchPrefix: head.split("/").slice(0, 2).join("/") + "/" } : linked); + } + } + return found; +} + +export async function releaseNestedRepositoryWorktrees(worktrees: LinkedNestedCheckout[]): Promise { + const warnings: string[] = []; + for (const worktree of worktrees) { + try { + await withNestedRepositoryBaseLock(worktree.base, async () => { + if (!await missing(worktree.checkout)) return; + await removeStaleRegistrations(worktree.base, worktree.checkout); + const refs = (await git(worktree.base, ["for-each-ref", "--format=%(objectname) %(refname)", `refs/heads/${worktree.branchPrefix}`])) + .split("\n").map(line => line.trim().split(" ")).filter(([sha, ref]) => /^[a-f0-9]{40}$/.test(sha ?? "") && ref); + const held = await checkedOutBranches(worktree.base); + for (const [sha, ref] of refs) { + if (held.has(ref!)) continue; + const unpushed = (await git(worktree.base, ["rev-list", "--max-count=1", sha!, "--not", "--remotes=origin"])).trim(); + if (unpushed) continue; + await git(worktree.base, ["update-ref", "-d", ref!, sha!]); + } + }); + } catch (error) { + warnings.push(`Could not release nested repository worktree "${worktree.checkout}": ${error instanceof Error ? error.message : String(error)}`); + } + } + return warnings; +} + +export async function repairMovedNestedRepositoryWorktree(base: string, moved: string) { + await withNestedRepositoryBaseLock(base, () => git(base, ["worktree", "repair", moved])); +} diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts index 3171e5e5a39..600d5e32a42 100644 --- a/server/src/services/workspace-runtime.ts +++ b/server/src/services/workspace-runtime.ts @@ -115,6 +115,7 @@ import { type PersistedExposureRowSnapshot, } from "./runtime-exposure/port-reservation.js"; import { resolveTailscaleDnsName } from "./runtime-exposure/tailscale-hostname.js"; +import { collectNestedRepositoryWorktrees, releaseNestedRepositoryWorktrees } from "./nested-repository-worktrees.js"; export function resolveShell(): string { const fallback = process.platform === "win32" ? "sh" : "/bin/sh"; @@ -3909,7 +3910,7 @@ export async function ensurePersistedExecutionWorkspaceAvailable(input: { const restoreContents = async () => { if (!restoration?.contents || !input.workspace.id) return; await restoreGitWorkspaceContents({ cwd, companyId: input.agent.companyId, workspaceId: input.workspace.id, - contents: restoration.contents, resolveGitAuth: async url => { + projectId: input.workspace.projectId, contents: restoration.contents, resolveGitAuth: async url => { const auth = await input.resolveGitAuth?.(url); if (!auth) return null; const source = auth.source; @@ -4292,6 +4293,10 @@ export async function cleanupExecutionWorkspaceArtifacts(input: { // Callers can require the workspace to match an assessed snapshot before // cleanup begins. Destructive paths recheck immediately before removal. await input.assertSafeToCleanup?.(); + const nestedWorktrees = workspacePath ? await collectNestedRepositoryWorktrees(workspacePath).catch((err) => { + warnings.push(`Could not inspect nested repository worktrees: ${err instanceof Error ? err.message : String(err)}`); + return []; + }) : []; let worktreeInstancePointer: WorktreeInstancePointer | null = null; let expectedWorktreeInstanceId: string | null = null; if (input.workspace.providerType === "git_worktree" && workspacePath) { @@ -4467,6 +4472,7 @@ export async function cleanupExecutionWorkspaceArtifacts(input: { const cleaned = !workspacePath || !(await directoryExists(workspacePath)); + if (cleaned) warnings.push(...await releaseNestedRepositoryWorktrees(nestedWorktrees)); return { cleanedPath: workspacePath,