diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 31196954cb9..0322ea4a90f 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -3072,6 +3072,20 @@ outcome, skip reason and reclaimed bytes. See [recoverable disk cleanup](recoverable-disk-cleanup.md#isolated-repository-sources) for the conditions and the verification it performs. +### Nested clone migration + +Tasks prepared before nested project repositories became worktrees of a shared +clone still hold a full standalone clone in each `.paperclip-repositories/` +checkout. Enable `PAPERCLIP_NESTED_CLONE_MIGRATION_ENABLED=true` (default off) to +let the terminal workspace reaper convert those checkouts in open tasks into +worktrees of the project's shared clone, keeping their exact state and local +history, and delete the standalone clones. Set +`PAPERCLIP_NESTED_CLONE_MIGRATION_DRY_RUN=true` to log candidates without changing +anything. Each attempt logs `execution_workspace.nested_clone_migration` with its +outcome, skip reason and reclaimed bytes. See +[recoverable disk cleanup](recoverable-disk-cleanup.md#converting-existing-standalone-nested-clones) +for the conditions, verification and crash recovery. + ### Terminal workspace compaction A finished task often keeps its worktree for a long time: the reaper archives a diff --git a/doc/recoverable-disk-cleanup.md b/doc/recoverable-disk-cleanup.md index c31954027e6..5642e560250 100644 --- a/doc/recoverable-disk-cleanup.md +++ b/doc/recoverable-disk-cleanup.md @@ -222,8 +222,9 @@ Git worktree of one shared clone per company, project and repository URL: the ma project checkout for that URL under the instance's `projects/` directory (the same clone that main task worktrees use when the repository is a task's primary source). Objects are stored once in that clone, so a new task adds only its working files. Repositories with a -configured local folder are still seeded as independent copies, and existing standalone -nested clones keep working unchanged; they are not migrated. +configured local folder are still seeded as independent copies. Existing standalone +nested clones keep working unchanged until the optional conversion below turns them into +worktrees of the shared clone. Each nested worktree checks out its own branch, `paperclip//`, where `` derives from the worktree path and `` is the remote's current default branch @@ -259,6 +260,92 @@ commit is on a remote-tracking ref; an unpushed branch stays in the shared clone nested worktree to `.paperclip-runtime/detached-repositories/` when its repository leaves the project repairs its registration. +### Converting existing standalone nested clones + +Tasks prepared before nested worktrees existed still hold a full standalone clone in each +`.paperclip-repositories/-` checkout, often of the same large repository many +times over. With `PAPERCLIP_NESTED_CLONE_MIGRATION_ENABLED=true`, the terminal workspace +reaper converts each such checkout in an open task into a worktree of the project's shared +clone and deletes the standalone clone. It holds the source lock and workspace lifecycle +lock that git cache expiration holds, the per-shared-clone lock that nested worktree +creation holds, and the expiration lock files (`index.lock`, `HEAD.lock` and the current +branch's ref lock) in the clone, so no run, +expiration, reopen, restore or other conversion against the same shared clone can +interleave. A checkout is converted only when: + +- no run, native finalization, environment lease or runtime service is still live for the + workspace, and it is not waiting for a reopen to be consumed (the issue may be open); +- its name is the managed checkout name of one of the project's repository workspaces + (`unknown_managed_checkout` otherwise), that repository has no configured local folder + (`local_source_checkout`), and the clone's `origin` is exactly the repository URL + (`foreign_origin`); +- the project has a verified shared clone for that URL (`no_shared_base`, + `shared_base_repo_mismatch`) whose local config runs no filters or commands + (`unsupported_base_git_config`); +- the clone holds only branches, tags and `origin` remote-tracking refs, no stash + (`nested_clone_has_stash`; stashes are not converted, because `refs/stash` would be shared + by every worktree of the shared clone), no worktrees of its own, no custom hooks, no local + config beyond the stock keys, `origin` and branch tracking, no `info/exclude` pattern the + shared clone lacks (`unsupported_info_exclude`), no in-progress merge, rebase or similar + state, no hidden index flags, and no commit that only another branch's reflog holds + (`unique_branch_reflog_commits`); +- nothing a restore point refuses (submodules, filtered paths, special files, untracked + nested repositories, oversized contents or history). + +The conversion fetches the shared clone, then brings the clone's history in under the +checkout's own prefix: the current branch becomes `paperclip//` at the same +commit, tracking `origin/` (or the branch's configured `origin` upstream); every +other local branch whose tip the shared clone's remote-tracking refs do not contain becomes +`paperclip//`, every such tag becomes `paperclip//tags/` at the +tagged commit, and every such `origin` remote-tracking ref (for example one whose branch was +deleted or force-pushed upstream) becomes `paperclip//remotes/origin/`. `` derives from the final checkout path, so later expiration and release +treat those branches exactly like a nested worktree's own branches: they block expiration +until pushed or saved and are never deleted while unpushed. A restore point of the clone, +captured against commits both repositories share and preserving those branch tips, its +`HEAD` reflog and `ORIG_HEAD`, is imported into the shared clone. Other tasks' branches and +worktrees are never touched; a name that already exists in the shared clone skips the +checkout (`nested_clone_ref_conflict`). + +The new worktree is added at `.clone-migrating` beside the clone, the restore point +is applied (index, worktree changes, untracked and ignored files, file modes), the `HEAD` +reflog and `ORIG_HEAD` are copied, and regenerable directories, untracked `.env` files and +`.claude/settings.local.json` are moved in by rename. The two must then match exactly +(HEAD, index, worktree, ignored files, modes, reflog, branch and every created ref). Any +difference or failure before the swap moves the items back, removes the new worktree and +the created branches, and leaves the clone untouched. The swap renames the clone to +`.clone-standalone`, moves the new worktree into `` with `git worktree move` +and runs `git worktree repair` in the shared clone. Then one database update records the +worktree in a saved restoration recipe that lists the checkout, and the old clone is +renamed to `.clone-trash` and deleted. + +Before it changes the shared clone, the conversion records +`nestedCloneMigrationPending.` in workspace metadata (shared clone, HEAD, created +refs, moved items and phase). The next pass uses it after a crash: before the swap it moves +items back, removes the new worktree and the created branches, then converts again; once +the clone has been renamed aside it finishes the swap and deletes the old clone, so work done +in the converted checkout after a crash is kept. A state that matches neither (for example a +new checkout created at `` while the clone was aside) is left alone with +`nested_migration_recovery_conflict`. The reaper also finishes a pending record after the +workspace is archived; if the task worktree is already gone, it forgets the record, removes +the stale registrations and deletes only created branches whose commits are on the remote. +Expiration treats the `.clone-` names as unexplained +files and keeps the workspace until the conversion settles. A dry run reports a pending +record without acting on it. + +Each attempt logs `event: "execution_workspace.nested_clone_migration"` with the checkout +name, outcome (`migrated`, `recovered`, `rolled_back`, `dry_run` or `skipped`), skip reason, +new branch, candidate bytes (the clone's `.git`) and reclaimed bytes. A workspace is +inspected again at most hourly, and a retry deadline it registered survives later sweeps +during that hour. A retry deadline is registered only while a pending record +exists or the skip comes from a live consumer, a busy lock or a changed workspace row; other +skips, including a change caught by verification, wait for a later sweep. The reaper looks +only at checkouts named after the project's configured repositories (plus pending records), +never at arbitrary directories, and attempts at most 8 per workspace per sweep. Pending +recoveries go first, and the rest rotate across sweeps. While a rotation has not yet visited +every checkout it registers a retry deadline; after a full rotation without progress it waits +for a later sweep, until a conversion succeeds or the set of candidate checkouts changes. `PAPERCLIP_NESTED_CLONE_MIGRATION_DRY_RUN=true` reports candidates +without changing anything and registers no deadlines. + Reopening the issue, or the next run that recreates the worktree, imports the bundle, re-adds the worktree at the saved commit, applies the restore point and verifies the result. The bundle is always imported, so history-only commits are present again as diff --git a/server/src/__tests__/nested-clone-migration.test.ts b/server/src/__tests__/nested-clone-migration.test.ts new file mode 100644 index 00000000000..e77222c4b9e --- /dev/null +++ b/server/src/__tests__/nested-clone-migration.test.ts @@ -0,0 +1,488 @@ +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { pathToFileURL } from "node:url"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { eq, sql } from "drizzle-orm"; +import { agents, companies, createDb, executionWorkspaceRestorePoints, executionWorkspaces, heartbeatRuns, issues, projects, projectWorkspaces } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { materializeManagedProjectWorkspace } from "../services/managed-repository-checkout.js"; +import { materializeIsolatedRemoteWorkspaceSource } from "../services/isolated-remote-workspace-source.js"; +import { realizeExecutionWorkspace } from "../services/workspace-runtime.js"; +import { executionWorkspaceService } from "../services/execution-workspaces.js"; +import { prepareProjectRepositoryWorkspaces } from "../services/heartbeat/workspaces.js"; +import { expireReconstructableGitWorkspace } from "../services/git-workspace-expiration.js"; +import { currentRestorePointSnapshot } from "../services/git-restore-point.js"; +import { nestedRepositoryBaseCandidates, nestedWorktreeBranchPrefix, readLinkedNestedCheckout, withNestedRepositoryBaseLock } from "../services/nested-repository-worktrees.js"; +import { listNestedCloneMigrationNames, migrateNestedRepositoryClone, nextNestedCloneMigrationBatch, type NestedCloneMigrationOptions } from "../services/nested-clone-migration.js"; +import { getStorageService } from "../storage/index.js"; + +const gitEnv = { ...process.env, GIT_AUTHOR_NAME: "Fixture", GIT_AUTHOR_EMAIL: "fixture@example.test", + GIT_COMMITTER_NAME: "Fixture", GIT_COMMITTER_EMAIL: "fixture@example.test", GIT_TERMINAL_PROMPT: "0" }; +const git = (cwd: string, ...args: string[]) => execFileSync("git", args, { cwd, env: gitEnv, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); +const exists = (target: string) => fs.lstat(target).then(() => true, () => false); +const DAY_MS = 24 * 60 * 60 * 1000; +const noAuth = async () => null; + +describe("nested clone migration batches", () => { + it("puts pending recoveries first, rotates through the rest and stops retrying an exhausted rotation until the list changes", () => { + const listed = Array.from({ length: 10 }, (_, index) => `repo-${index.toString(16).padStart(12, "0")}`); + const first = nextNestedCloneMigrationBatch(listed, [listed[9]!], undefined, 4); + expect(first).toMatchObject({ names: [listed[9], listed[0], listed[1], listed[2]], overflow: true, retryOverflow: true }); + const second = nextNestedCloneMigrationBatch(listed, [], first.rotation, 4); + expect(second).toMatchObject({ names: [listed[3], listed[4], listed[5], listed[6]], retryOverflow: true }); + const third = nextNestedCloneMigrationBatch(listed, [], second.rotation, 4); + expect(third).toMatchObject({ names: [listed[7], listed[8], listed[0], listed[1]], retryOverflow: false }); + expect([...third.rotation.visited]).toEqual([]); + const fourth = nextNestedCloneMigrationBatch(listed, [], third.rotation, 4); + expect(fourth).toMatchObject({ names: listed.slice(0, 4), retryOverflow: false }); + const grown = [...listed, "repo-aaaaaaaaaaaa"]; + expect(nextNestedCloneMigrationBatch(grown, [], fourth.rotation, 4)).toMatchObject({ retryOverflow: true }); + expect(nextNestedCloneMigrationBatch(listed.slice(0, 3), [], undefined, 4)).toMatchObject({ names: listed.slice(0, 3), overflow: false, retryOverflow: false }); + }); +}); + +const support = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = support.supported ? describe : describe.skip; +describeEmbeddedPostgres("nested repository clone migration", () => { + let db: ReturnType; + let database: Awaited>; + let root: string; + const previousHome = process.env.PAPERCLIP_HOME; + const previousInstance = process.env.PAPERCLIP_INSTANCE_ID; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("nested-clone-migration-"); + db = createDb(database.connectionString); + root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-nested-migration-")); + process.env.PAPERCLIP_HOME = path.join(root, "home"); + process.env.PAPERCLIP_INSTANCE_ID = "default"; + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "51".repeat(32)); + }); + afterEach(async () => { + await db.delete(executionWorkspaceRestorePoints); + await db.execute(sql`TRUNCATE TABLE companies CASCADE`); + }); + afterAll(async () => { + vi.unstubAllEnvs(); + if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome; + if (previousInstance === undefined) delete process.env.PAPERCLIP_INSTANCE_ID; else process.env.PAPERCLIP_INSTANCE_ID = previousInstance; + await db.$client.end({ timeout: 1 }); await database.cleanup(); await fs.rm(root, { recursive: true, force: true }); + }); + + async function repository(location: string, name: string, files: Record) { + const author = path.join(location, `${name}-author`), remote = path.join(location, `${name}.git`); + await fs.mkdir(author, { recursive: true }); + git(author, "init", "--initial-branch=main"); + for (const [file, contents] of Object.entries(files)) await fs.writeFile(path.join(author, file), contents); + git(author, "add", "."); git(author, "commit", "-m", `${name} A`); + git(location, "clone", "--bare", author, remote); + return { author, remote, url: pathToFileURL(remote).href, commit: git(author, "rev-parse", "HEAD") }; + } + + async function fixture(options: { issueStatus?: "in_progress" | "done"; sharedBase?: boolean } = {}) { + const companyId = randomUUID(), projectId = randomUUID(); + const location = path.join(root, companyId); + const main = await repository(location, "remote", { "marker.txt": "commit A\n", ".gitignore": "node_modules/\n.claude/settings.local.json\n" }); + const nested = await repository(location, "nested", { "nested.txt": "nested A\n", "remove-me.txt": "tracked\n", + ".gitignore": "node_modules/\n.env\n.claude/settings.local.json\n*.log\n" }); + await db.insert(companies).values({ id: companyId, name: "Nested migration fixture", issuePrefix: `NM${companyId.slice(0, 6)}` }); + await db.insert(projects).values({ id: projectId, companyId, name: "Fixture project" }); + const [source] = await db.insert(projectWorkspaces).values({ companyId, projectId, name: "Nested repository", repoUrl: nested.url, repoRef: "main" }).returning(); + const candidates = nestedRepositoryBaseCandidates({ companyId, projectId, repoUrl: nested.url }); + if (options.sharedBase !== false) await materializeManagedProjectWorkspace(candidates[0]!, { repoUrl: nested.url }); + const done = options.issueStatus === "done"; + const task = async (label: string) => { + const issueId = randomUUID(), workspaceId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, projectId, title: `Task ${label}`, status: done ? "done" : "in_progress", + ...(done ? { completedAt: new Date(Date.now() - 2 * DAY_MS) } : {}) }); + const baseCwd = await materializeIsolatedRemoteWorkspaceSource({ companyId, issueId, repoUrl: main.url, resolveGitAuth: noAuth }); + const base = { baseCwd, source: "task_session" as const, projectId: null, workspaceId: null, repoUrl: main.url, repoRef: "origin/main" }; + const realized = await realizeExecutionWorkspace({ base, issue: { id: issueId, identifier: `NM-${label}`, title: "Fixture" }, + agent: { id: null, name: "Fixture", companyId }, skipHostProvision: true, + config: { workspaceStrategy: { type: "git_worktree", branchTemplate: `task-${label}`, worktreeParentDir: path.join(baseCwd, ".paperclip", "worktrees"), baseRef: "origin/main" } } }); + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, projectId, sourceIssueId: issueId, mode: "isolated_workspace", + strategyType: "git_worktree", name: `Task ${label}`, status: "active", providerType: "local_fs", cwd: realized.cwd, providerRef: realized.cwd, + repoUrl: main.url, baseRef: "origin/main", branchName: realized.branchName, + metadata: { createdByRuntime: realized.branchCreatedByRuntime, gitBranchOwnershipVersion: 1, baseRefSha: realized.baseRefSha, + isolatedRemoteSource: { schema: "paperclip.isolated-remote-source.v1", companyId, issueId, repoUrl: main.url } } }); + await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, issueId)); + const [prepared] = await prepareProjectRepositoryWorkspaces({ cwd: realized.cwd, anchorRepoUrl: main.url, workspaces: [source!], resolveGitAuth: noAuth }); + const checkout = await fs.realpath(prepared!.cwd); + const row = () => db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId)).then(rows => rows[0]!); + const migrate = async (migrationOptions: NestedCloneMigrationOptions = {}) => + migrateNestedRepositoryClone(db, await row(), path.basename(checkout), { resolveGitAuth: noAuth, ...migrationOptions }); + const expire = async () => expireReconstructableGitWorkspace(db, await row(), { dryRun: false, expireBase: false, now: new Date(), + restorePoint: { storage: getStorageService(), notBefore: new Date(Date.now() - DAY_MS), keyDurable: true } }); + const reopen = () => executionWorkspaceService(db).reopenClosedIsolatedExecutionWorkspaceForIssue({ workspaceId, + issue: { id: issueId, companyId, projectId: null }, actor: { actorType: "user", agentId: null }, + authorizationActor: { type: "board", source: "local_implicit", isInstanceAdmin: true } }); + return { issueId, workspaceId, cwd: realized.cwd, checkout, name: path.basename(checkout), prefix: await nestedWorktreeBranchPrefix(checkout), + row, migrate, expire, reopen }; + }; + const first = await task("one"); + return { companyId, projectId, main, nested, candidates, base: candidates[0]!, task, ...first }; + } + + async function nestedWork(checkout: string) { + await fs.writeFile(path.join(checkout, "nested.txt"), "nested B (unpushed)\n"); + git(checkout, "commit", "-qam", "Nested B unpushed"); + const unpushed = git(checkout, "rev-parse", "HEAD"); + git(checkout, "checkout", "-q", "-b", "side"); + await fs.writeFile(path.join(checkout, "side.txt"), "side branch work\n"); + git(checkout, "add", "side.txt"); git(checkout, "commit", "-qm", "Side work"); + const side = git(checkout, "rev-parse", "HEAD"); + git(checkout, "tag", "local-tag"); + git(checkout, "checkout", "-q", "main"); + git(checkout, "tag", "-a", "-m", "annotated local release", "local-release"); + await fs.writeFile(path.join(checkout, "nested.txt"), "staged edit\n"); + git(checkout, "add", "nested.txt"); + await fs.writeFile(path.join(checkout, "nested.txt"), "unstaged edit\n"); + await fs.rm(path.join(checkout, "remove-me.txt")); + await fs.writeFile(path.join(checkout, "draft.md"), "untracked work\n"); + await fs.chmod(path.join(checkout, "draft.md"), 0o600); + await fs.writeFile(path.join(checkout, "build.log"), "ignored output\n"); + await fs.mkdir(path.join(checkout, "node_modules", "pkg"), { recursive: true }); + await fs.writeFile(path.join(checkout, "node_modules", "pkg", "index.js"), "module.exports = 1;\n"); + await fs.writeFile(path.join(checkout, ".env"), "LOCAL_SETTING=1\n"); + await fs.mkdir(path.join(checkout, ".claude")); + await fs.writeFile(path.join(checkout, ".claude", "settings.local.json"), "{\"local\":true}\n"); + return { unpushed, side }; + } + + const gitView = (cwd: string) => ({ head: git(cwd, "rev-parse", "HEAD"), status: git(cwd, "status", "--porcelain=v1", "--untracked-files=all"), + reflog: git(cwd, "reflog", "show", "--format=%H", "HEAD"), staged: git(cwd, "show", ":nested.txt") }); + const ownedRefs = (base: string, prefix: string) => git(base, "for-each-ref", "--format=%(refname) %(objectname)", `refs/heads/${prefix}`); + const worktreePaths = (base: string) => git(base, "worktree", "list", "--porcelain").split("\n").filter(line => line.startsWith("worktree ")).map(line => line.slice(9)); + const staging = (checkout: string) => [".clone-migrating", ".clone-standalone", ".clone-trash"].map(suffix => `${checkout}${suffix}`); + + async function expectUntouched(f: Awaited>, before: Awaited>, checkout = f.checkout, prefix = f.prefix) { + expect((await fs.lstat(path.join(checkout, ".git"))).isDirectory()).toBe(true); + expect(await currentRestorePointSnapshot(checkout)).toEqual(before); + for (const target of staging(checkout)) expect(await exists(target)).toBe(false); + expect(ownedRefs(f.base, prefix)).toBe(""); + expect(worktreePaths(f.base)).toHaveLength(1); + expect((await f.row()).metadata?.nestedCloneMigrationPending).toBeUndefined(); + } + + it("converts a standalone nested clone into a worktree of the shared clone with its exact state and history", async () => { + const f = await fixture(); + const { unpushed, side } = await nestedWork(f.checkout); + const before = await currentRestorePointSnapshot(f.checkout); + const view = gitView(f.checkout); + const realBase = await fs.realpath(f.base); + expect(await listNestedCloneMigrationNames(db, await f.row())).toEqual([f.name]); + + const result = await f.migrate(); + expect(result).toMatchObject({ outcome: "migrated", reason: null, branchName: `${f.prefix}main` }); + expect(result.reclaimedBytes).toBeGreaterThan(0); + expect(result.candidateBytes).toBeGreaterThan(0); + + expect((await fs.lstat(path.join(f.checkout, ".git"))).isFile()).toBe(true); + expect(git(f.checkout, "rev-parse", "--path-format=absolute", "--git-common-dir")).toBe(path.join(realBase, ".git")); + expect(await readLinkedNestedCheckout(f.checkout)).toMatchObject({ base: realBase, checkout: f.checkout }); + expect(worktreePaths(f.base)).toContain(f.checkout); + expect(await currentRestorePointSnapshot(f.checkout)).toEqual(before); + expect(gitView(f.checkout)).toEqual(view); + expect(view.head).toBe(unpushed); + expect(git(f.checkout, "symbolic-ref", "--short", "HEAD")).toBe(`${f.prefix}main`); + expect(git(f.checkout, "rev-parse", "--abbrev-ref", "@{upstream}")).toBe("origin/main"); + expect(await fs.readFile(path.join(f.checkout, "nested.txt"), "utf8")).toBe("unstaged edit\n"); + expect(await exists(path.join(f.checkout, "remove-me.txt"))).toBe(false); + expect((await fs.stat(path.join(f.checkout, "draft.md"))).mode & 0o777).toBe(0o600); + expect(await fs.readFile(path.join(f.checkout, "build.log"), "utf8")).toBe("ignored output\n"); + expect(await fs.readFile(path.join(f.checkout, "node_modules", "pkg", "index.js"), "utf8")).toBe("module.exports = 1;\n"); + expect(await fs.readFile(path.join(f.checkout, ".env"), "utf8")).toBe("LOCAL_SETTING=1\n"); + expect(await fs.readFile(path.join(f.checkout, ".claude", "settings.local.json"), "utf8")).toBe("{\"local\":true}\n"); + + expect(git(f.base, "rev-parse", `refs/heads/${f.prefix}main`)).toBe(unpushed); + expect(git(f.base, "rev-parse", `refs/heads/${f.prefix}side`)).toBe(side); + expect(git(f.base, "rev-parse", `refs/heads/${f.prefix}tags/local-tag`)).toBe(side); + expect(git(f.base, "rev-parse", `refs/heads/${f.prefix}tags/local-release`)).toBe(unpushed); + expect(git(f.base, "for-each-ref", "--format=%(refname)", "refs/heads").split("\n").filter(ref => !ref.startsWith(`refs/heads/${f.prefix}`))).toEqual(["refs/heads/main"]); + expect(git(f.base, "status", "--porcelain")).toBe(""); + for (const target of staging(f.checkout)) expect(await exists(target)).toBe(false); + expect((await f.row()).metadata?.nestedCloneMigrationPending).toBeUndefined(); + expect(await f.migrate()).toMatchObject({ outcome: "skipped", reason: "already_worktree_backed" }); + expect(await listNestedCloneMigrationNames(db, await f.row())).toEqual([]); + }); + + it("keeps commits that only the clone's remote-tracking refs still hold", async () => { + const f = await fixture(); + git(f.nested.author, "checkout", "-q", "-b", "squashed-away"); + await fs.writeFile(path.join(f.nested.author, "feature.txt"), "feature\n"); + git(f.nested.author, "add", "."); git(f.nested.author, "commit", "-qm", "Feature"); + const feature = git(f.nested.author, "rev-parse", "HEAD"); + git(f.nested.author, "push", "-q", f.nested.remote, "squashed-away"); + git(f.checkout, "fetch", "-q", "origin"); + git(f.nested.remote, "branch", "-D", "squashed-away"); + git(f.nested.remote, "reflog", "expire", "--expire=now", "--all"); + git(f.nested.remote, "gc", "-q", "--prune=now"); + expect(() => git(f.base, "cat-file", "-e", `${feature}^{commit}`)).toThrow(); + expect(await f.migrate()).toMatchObject({ outcome: "migrated" }); + expect(git(f.base, "rev-parse", `refs/heads/${f.prefix}remotes/origin/squashed-away`)).toBe(feature); + expect(git(f.checkout, "cat-file", "-t", feature)).toBe("commit"); + }); + + it("finishes an interrupted conversion after the workspace is archived and forgets one whose workspace is gone", async () => { + const f = await fixture(); + const { unpushed } = await nestedWork(f.checkout); + const before = await currentRestorePointSnapshot(f.checkout); + expect(await f.migrate({ hooks: { afterMove: async () => { throw new Error("simulated crash"); } } })).toMatchObject({ outcome: "skipped" }); + await db.update(executionWorkspaces).set({ status: "archived", closedAt: new Date() }).where(eq(executionWorkspaces.id, f.workspaceId)); + const service = executionWorkspaceService(db, { nestedCloneMigrationEnabled: true }); + await service.sweepTerminalWorkspaces(); + expect((await f.row()).metadata?.nestedCloneMigrationPending).toBeUndefined(); + expect(await exists(`${f.checkout}.clone-standalone`)).toBe(false); + expect(git(f.checkout, "rev-parse", "HEAD")).toBe(unpushed); + expect(await currentRestorePointSnapshot(f.checkout)).toEqual(before); + expect(await service.nextTerminalWorkspaceSweepAt()).toBeNull(); + + const second = await f.task("two"); + await nestedWork(second.checkout); + expect(await second.migrate({ hooks: { afterAside: async () => { throw new Error("simulated crash"); } } })).toMatchObject({ outcome: "skipped" }); + await db.update(executionWorkspaces).set({ status: "archived", closedAt: new Date() }).where(eq(executionWorkspaces.id, second.workspaceId)); + await fs.rm(second.cwd, { recursive: true, force: true }); + expect(await second.migrate()).toMatchObject({ outcome: "rolled_back", reason: "workspace_removed" }); + expect((await second.row()).metadata?.nestedCloneMigrationPending).toBeUndefined(); + expect(git(f.base, "rev-parse", `refs/heads/${second.prefix}main`)).toBeTruthy(); + }); + + it("skips a nested clone with a stash without changing anything", async () => { + const f = await fixture(); + await fs.writeFile(path.join(f.checkout, "nested.txt"), "stashed edit\n"); + git(f.checkout, "stash", "-q"); + const before = await currentRestorePointSnapshot(f.checkout); + expect(await f.migrate()).toMatchObject({ outcome: "skipped", reason: "nested_clone_has_stash" }); + await expectUntouched(f, before); + expect(git(f.checkout, "stash", "list")).not.toBe(""); + }); + + it("skips a nested clone whose local excludes the shared clone would not apply", async () => { + const f = await fixture(); + await fs.appendFile(path.join(f.checkout, ".git", "info", "exclude"), "\nscratch/\n"); + await fs.mkdir(path.join(f.checkout, "scratch")); + await fs.writeFile(path.join(f.checkout, "scratch", "notes.txt"), "locally excluded\n"); + const before = await currentRestorePointSnapshot(f.checkout); + expect(await f.migrate()).toMatchObject({ outcome: "skipped", reason: "unsupported_info_exclude" }); + await expectUntouched(f, before); + }); + + it("never checks out through filters planted in the shared clone during the conversion", async () => { + const f = await fixture(); + await fs.writeFile(path.join(f.checkout, ".gitattributes"), "*.dat filter=evil\n"); + await fs.writeFile(path.join(f.checkout, "blob.dat"), "payload\n"); + git(f.checkout, "add", ".gitattributes", "blob.dat"); git(f.checkout, "commit", "-qm", "Attributes"); + await fs.writeFile(path.join(f.checkout, "draft.md"), "untracked work\n"); + const marker = path.join(path.dirname(f.nested.remote), "smudge-ran"); + const before = await currentRestorePointSnapshot(f.checkout); + const result = await f.migrate({ hooks: { beforeWorktreeAdd: async () => { + git(f.base, "config", "filter.evil.smudge", `sh -c 'touch ${marker}; cat'`); + git(f.base, "config", "filter.evil.required", "true"); + } } }); + expect(result).toMatchObject({ outcome: "skipped", reason: "unsupported_base_git_config" }); + expect(await exists(marker)).toBe(false); + git(f.base, "config", "--remove-section", "filter.evil"); + await expectUntouched(f, before); + }); + + it("skips while a run is live", async () => { + const f = await fixture(); + await nestedWork(f.checkout); + const agentId = randomUUID(); + await db.insert(agents).values({ id: agentId, companyId: f.companyId, name: "Fixture", role: "engineer", adapterType: "process" }); + await db.insert(heartbeatRuns).values({ id: randomUUID(), companyId: f.companyId, agentId, issueId: f.issueId, status: "running" }); + const before = await currentRestorePointSnapshot(f.checkout); + expect(await f.migrate()).toMatchObject({ outcome: "skipped", reason: "live_or_unresolved_consumer" }); + await expectUntouched(f, before); + }); + + it("skips a nested clone whose origin differs from the project repository", async () => { + const f = await fixture(); + await nestedWork(f.checkout); + git(f.checkout, "remote", "set-url", "origin", f.main.url); + const before = await currentRestorePointSnapshot(f.checkout); + expect(await f.migrate()).toMatchObject({ outcome: "skipped", reason: "foreign_origin" }); + await expectUntouched(f, before); + }); + + it("skips a checkout that is not a configured project repository", async () => { + const f = await fixture(); + const stray = path.join(path.dirname(f.checkout), "stray-0123456789ab"); + git(path.dirname(stray), "clone", "-q", f.nested.url, stray); + const before = await currentRestorePointSnapshot(stray); + for (let index = 0; index < 20; index += 1) await fs.mkdir(path.join(path.dirname(f.checkout), `agent-${index.toString(16).padStart(12, "0")}`, ".git"), { recursive: true }); + expect(await listNestedCloneMigrationNames(db, await f.row())).toEqual([f.name]); + expect(await migrateNestedRepositoryClone(db, await f.row(), "stray-0123456789ab", { resolveGitAuth: noAuth })) + .toMatchObject({ outcome: "skipped", reason: "unknown_managed_checkout" }); + await expectUntouched(f, before, stray, await nestedWorktreeBranchPrefix(stray)); + }); + + it("skips when the project has no shared clone", async () => { + const f = await fixture({ sharedBase: false }); + expect(await f.migrate()).toMatchObject({ outcome: "skipped", reason: "no_shared_base" }); + expect((await fs.lstat(path.join(f.checkout, ".git"))).isDirectory()).toBe(true); + }); + + it("rolls back and keeps the clone when it changes between capture and verification", async () => { + const f = await fixture(); + await nestedWork(f.checkout); + const before = await currentRestorePointSnapshot(f.checkout); + const result = await f.migrate({ hooks: { beforeVerify: async ({ from, to }) => { + expect(await exists(path.join(to, "node_modules", "pkg", "index.js"))).toBe(true); + await fs.writeFile(path.join(from, "late.txt"), "written by a concurrent run\n"); + } } }); + expect(result).toMatchObject({ outcome: "skipped", reason: "snapshot_mismatch" }); + await fs.rm(path.join(f.checkout, "late.txt")); + await expectUntouched(f, before); + expect(await fs.readFile(path.join(f.checkout, "node_modules", "pkg", "index.js"), "utf8")).toBe("module.exports = 1;\n"); + expect(await fs.readFile(path.join(f.checkout, ".env"), "utf8")).toBe("LOCAL_SETTING=1\n"); + }); + + it("serializes two tasks converting against the same shared clone and keeps their branches apart", async () => { + const f = await fixture(); + const second = await f.task("two"); + const one = await nestedWork(f.checkout); + await fs.writeFile(path.join(second.checkout, "nested.txt"), "second task work\n"); + git(second.checkout, "commit", "-qam", "Second task unpushed"); + const twoHead = git(second.checkout, "rev-parse", "HEAD"); + let active = 0, peak = 0, reached = 0; + const hooks = { beforeVerify: async () => { + reached += 1; active += 1; peak = Math.max(peak, active); + await new Promise(resolve => setTimeout(resolve, 200)); + active -= 1; + } }; + let release!: () => void; + const held = withNestedRepositoryBaseLock(f.base, () => new Promise(resolve => { release = resolve; })); + const running = Promise.all([f.migrate({ hooks }), second.migrate({ hooks })]); + await new Promise(resolve => setTimeout(resolve, 3_000)); + expect(reached).toBe(0); + expect((await fs.lstat(path.join(f.checkout, ".git"))).isDirectory()).toBe(true); + expect((await fs.lstat(path.join(second.checkout, ".git"))).isDirectory()).toBe(true); + expect(ownedRefs(f.base, f.prefix)).toBe(""); + release(); + await held; + const [a, b] = await running; + expect(a).toMatchObject({ outcome: "migrated" }); + expect(b).toMatchObject({ outcome: "migrated" }); + expect(peak).toBe(1); + expect(f.prefix).not.toBe(second.prefix); + expect(git(f.checkout, "rev-parse", "HEAD")).toBe(one.unpushed); + expect(git(second.checkout, "rev-parse", "HEAD")).toBe(twoHead); + expect(ownedRefs(f.base, second.prefix)).toBe(`refs/heads/${second.prefix}main ${twoHead}`); + expect(ownedRefs(f.base, f.prefix).split("\n")).toHaveLength(4); + expect(worktreePaths(f.base)).toEqual(expect.arrayContaining([f.checkout, second.checkout])); + }); + + it.each(["afterAside", "afterMove", "afterFinalize"] as const)("recovers a conversion interrupted at %s on the next pass", async (step) => { + const f = await fixture(); + const { unpushed } = await nestedWork(f.checkout); + const before = await currentRestorePointSnapshot(f.checkout); + const crashed = await f.migrate({ hooks: { [step]: async () => { throw new Error("simulated crash"); } } }); + expect(crashed).toMatchObject({ outcome: "skipped" }); + expect(await exists(`${f.checkout}.clone-standalone`)).toBe(true); + expect((await f.row()).metadata?.nestedCloneMigrationPending).toMatchObject({ [f.name]: { phase: step === "afterFinalize" ? "cleanup" : "swap" } }); + expect(await listNestedCloneMigrationNames(db, await f.row())).toEqual([f.name]); + expect(await f.migrate({ dryRun: true })).toMatchObject({ outcome: "dry_run", reason: "interrupted_migration_pending" }); + expect(await exists(`${f.checkout}.clone-standalone`)).toBe(true); + + const recovered = await f.migrate(); + expect(recovered).toMatchObject({ outcome: "recovered", reason: null, branchName: `${f.prefix}main` }); + expect(recovered.reclaimedBytes).toBeGreaterThan(0); + expect((await fs.lstat(path.join(f.checkout, ".git"))).isFile()).toBe(true); + expect(await currentRestorePointSnapshot(f.checkout)).toEqual(before); + expect(git(f.checkout, "rev-parse", "HEAD")).toBe(unpushed); + for (const target of staging(f.checkout)) expect(await exists(target)).toBe(false); + expect((await f.row()).metadata?.nestedCloneMigrationPending).toBeUndefined(); + expect(worktreePaths(f.base)).toEqual([await fs.realpath(f.base), f.checkout]); + }); + + it("undoes a conversion interrupted before the swap and then completes it", async () => { + const f = await fixture(); + const { unpushed } = await nestedWork(f.checkout); + const before = await currentRestorePointSnapshot(f.checkout); + const realBase = await fs.realpath(f.base); + const staged = `${f.checkout}.clone-migrating`; + const branch = `${f.prefix}main`; + git(realBase, "fetch", "-q", f.checkout, `refs/heads/main:refs/heads/${branch}`); + git(realBase, "worktree", "add", "-q", staged, branch); + await fs.rename(path.join(f.checkout, "node_modules"), path.join(staged, "node_modules")); + const pending = { base: realBase, head: unpushed, branch, refs: [{ ref: `refs/heads/${branch}`, commit: unpushed, upstream: "main" }], + moves: ["node_modules"], phase: "transfer" }; + const current = await f.row(); + await db.update(executionWorkspaces).set({ metadata: { ...current.metadata, nestedCloneMigrationPending: { [f.name]: pending } } }) + .where(eq(executionWorkspaces.id, f.workspaceId)); + expect(await f.migrate({ dryRun: true })).toMatchObject({ outcome: "dry_run", reason: "interrupted_migration_pending" }); + expect(await exists(path.join(staged, "node_modules"))).toBe(true); + + const result = await f.migrate(); + expect(result).toMatchObject({ outcome: "migrated", reason: null }); + expect(await currentRestorePointSnapshot(f.checkout)).toEqual(before); + expect(await fs.readFile(path.join(f.checkout, "node_modules", "pkg", "index.js"), "utf8")).toBe("module.exports = 1;\n"); + for (const target of staging(f.checkout)) expect(await exists(target)).toBe(false); + expect(worktreePaths(f.base)).toEqual([realBase, f.checkout]); + expect((await f.row()).metadata?.nestedCloneMigrationPending).toBeUndefined(); + }); + + it("expires and restores a converted checkout and protects its unpushed work", async () => { + const f = await fixture({ issueStatus: "done" }); + await fs.writeFile(path.join(f.checkout, "nested.txt"), "nested B (unpushed)\n"); + git(f.checkout, "commit", "-qam", "Nested B unpushed"); + const unpushed = git(f.checkout, "rev-parse", "HEAD"); + await fs.writeFile(path.join(f.checkout, "draft.md"), "nested untracked work\n"); + expect(await f.migrate()).toMatchObject({ outcome: "migrated" }); + const realBase = await fs.realpath(f.base); + + expect(await f.expire()).toMatchObject({ expired: true, restorePoint: true, skipReason: null }); + expect(await exists(f.cwd)).toBe(false); + const recipe = (await f.row()).metadata?.gitCacheRestoration as Record; + expect(recipe.contents.checkouts[0]).toMatchObject({ name: f.name, commit: unpushed, branchName: `${f.prefix}main`, worktree: { base: realBase } }); + expect(await f.reopen()).toMatchObject({ ok: true, reopened: true }); + expect(git(f.checkout, "rev-parse", "--path-format=absolute", "--git-common-dir")).toBe(path.join(realBase, ".git")); + expect(git(f.checkout, "symbolic-ref", "--short", "HEAD")).toBe(`${f.prefix}main`); + expect(git(f.checkout, "rev-parse", "HEAD")).toBe(unpushed); + expect(await fs.readFile(path.join(f.checkout, "draft.md"), "utf8")).toBe("nested untracked work\n"); + + const second = await f.task("two"); + await nestedWork(second.checkout); + await fs.rm(path.join(second.checkout, "node_modules"), { recursive: true }); + expect(await second.migrate()).toMatchObject({ outcome: "migrated" }); + expect(await second.expire()).toMatchObject({ expired: false, skipReason: "unpushed_managed_checkout_branches" }); + expect(await exists(second.checkout)).toBe(true); + }); + + it("is disabled by default, changes nothing in dry run, and runs from the reaper when enabled", async () => { + const f = await fixture(); + const { unpushed } = await nestedWork(f.checkout); + const before = await currentRestorePointSnapshot(f.checkout); + const disabled = executionWorkspaceService(db); + await disabled.sweepTerminalWorkspaces(); + await expectUntouched(f, before); + expect(await disabled.nextTerminalWorkspaceSweepAt()).toBeNull(); + + const dryRun = executionWorkspaceService(db, { nestedCloneMigrationEnabled: true, nestedCloneMigrationDryRun: true }); + await dryRun.sweepTerminalWorkspaces(); + await expectUntouched(f, before); + expect(await dryRun.nextTerminalWorkspaceSweepAt()).toBeNull(); + + const agentId = randomUUID(); + await db.insert(agents).values({ id: agentId, companyId: f.companyId, name: "Fixture", role: "engineer", adapterType: "process" }); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId, issueId: f.issueId, status: "running" }).returning(); + const busy = executionWorkspaceService(db, { nestedCloneMigrationEnabled: true }); + await busy.sweepTerminalWorkspaces(); + await expectUntouched(f, before); + const retryAt = await busy.nextTerminalWorkspaceSweepAt(); + expect(retryAt).toBeGreaterThan(Date.now()); + await busy.sweepTerminalWorkspaces(); + expect(await busy.nextTerminalWorkspaceSweepAt()).toBe(retryAt); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run!.id)); + + const enabled = executionWorkspaceService(db, { nestedCloneMigrationEnabled: true }); + await enabled.sweepTerminalWorkspaces(); + expect((await fs.lstat(path.join(f.checkout, ".git"))).isFile()).toBe(true); + expect(git(f.checkout, "rev-parse", "HEAD")).toBe(unpushed); + expect(await currentRestorePointSnapshot(f.checkout)).toEqual(before); + expect(await enabled.nextTerminalWorkspaceSweepAt()).toBeNull(); + }); +}); diff --git a/server/src/config.ts b/server/src/config.ts index d9acf20c592..e63b66ee0b0 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -84,6 +84,8 @@ export interface Config { gitCacheExpirationDryRun: boolean; isolatedSourceRehomeEnabled: boolean; isolatedSourceRehomeDryRun: boolean; + nestedCloneMigrationEnabled: boolean; + nestedCloneMigrationDryRun: boolean; workspaceReaperCooldownDays: number; workspaceCompactionDelayHours: number | null; skillVersionRetentionEnabled: boolean; @@ -400,6 +402,8 @@ export function loadConfig(): Config { gitCacheExpirationDryRun: process.env.PAPERCLIP_GIT_CACHE_EXPIRATION_DRY_RUN !== "false", isolatedSourceRehomeEnabled: process.env.PAPERCLIP_ISOLATED_SOURCE_REHOME_ENABLED === "true", isolatedSourceRehomeDryRun: process.env.PAPERCLIP_ISOLATED_SOURCE_REHOME_DRY_RUN === "true", + nestedCloneMigrationEnabled: process.env.PAPERCLIP_NESTED_CLONE_MIGRATION_ENABLED === "true", + nestedCloneMigrationDryRun: process.env.PAPERCLIP_NESTED_CLONE_MIGRATION_DRY_RUN === "true", workspaceReaperCooldownDays, workspaceCompactionDelayHours, skillVersionRetentionEnabled: process.env.PAPERCLIP_SKILL_VERSION_RETENTION_ENABLED !== "false", diff --git a/server/src/index.ts b/server/src/index.ts index 5eb3d88c599..814c63ebd9b 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1487,6 +1487,8 @@ async function startServerWithDatabaseTeardown( gitCacheExpirationDryRun: config.gitCacheExpirationDryRun, isolatedSourceRehomeEnabled: config.isolatedSourceRehomeEnabled, isolatedSourceRehomeDryRun: config.isolatedSourceRehomeDryRun, + nestedCloneMigrationEnabled: config.nestedCloneMigrationEnabled, + nestedCloneMigrationDryRun: config.nestedCloneMigrationDryRun, gitRestorePoints: config.gitRestorePointsEnabled ? { storage: storageService, terminalDelayMs: config.gitRestorePointTerminalDelayDays * 24 * 60 * 60 * 1000, diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index 0b922eeecc2..d5c456c8b06 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -7,6 +7,8 @@ import { assertExecutionWorkspaceRecordAccess, assertTaskWorkspaceExecutionAcces import { verifiedRecordedIsolatedRemoteSource, withIsolatedRepositorySourceLock } from "./isolated-remote-workspace-source.js"; import { expireReconstructableGitWorkspace, gitWorkspaceLastActivityAt, hasPendingGitExpiration } from "./git-workspace-expiration.js"; import { hasPendingIsolatedRepositoryRehomeCleanup, isIsolatedRepositoryRehomeCandidate, rehomeExpiredIsolatedRestoration, rehomeIsolatedRepositoryWorkspace } from "./isolated-repository-rehome.js"; +import { hasPendingNestedCloneMigration, isNestedCloneMigrationCandidate, nextNestedCloneMigrationBatch, pendingNestedCloneMigrationNames, listNestedCloneMigrationNames, migrateNestedRepositoryClone, + NESTED_CLONE_MIGRATION_RETRY_REASONS, type NestedCloneMigrationRotation } from "./nested-clone-migration.js"; import type { StorageService } from "../storage/types.js"; import { readGitCacheRestoration, restoreGitCacheBase, recoverGitCacheExpirationLocks, GitCacheRestorationFailure, gitRestorePointExpired } from "./git-workspace-restoration.js"; @@ -304,6 +306,8 @@ export type ExecutionWorkspaceServiceOptions = { gitCacheExpirationDryRun?: boolean; isolatedSourceRehomeEnabled?: boolean; isolatedSourceRehomeDryRun?: boolean; + nestedCloneMigrationEnabled?: boolean; + nestedCloneMigrationDryRun?: boolean; gitRestorePoints?: { storage: StorageService; terminalDelayMs: number; keyDurable: boolean; idleMs?: number | null }; terminalWorkspaceInspectionBackoffMs?: number; resolvePullRequestDetails?: PullRequestMergeDetailsResolver; @@ -1748,6 +1752,10 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic const isolatedRehomeDeadlines = new Map(); const isolatedRehomeNotBefore = new Map(); const ISOLATED_REHOME_RETRY_MS = 60 * 60 * 1000; + const nestedCloneMigrationDeadlines = new Map(); + const nestedCloneMigrationNotBefore = new Map(); + const nestedCloneMigrationRetryAt = new Map(); + const nestedCloneMigrationRotations = new Map(); let terminalSweepRescanRequested = false; // The reaper freezes an upper bound on updatedAt at the start of each // rotation. The scan only reads candidates at or below the bound, so a steady @@ -3253,6 +3261,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic sql`(${executionWorkspaces.metadata}->>'workspaceProviderRemoved') IS DISTINCT FROM 'true'`), opts.gitCacheExpirationEnabled ? and(inArray(executionWorkspaces.status, ["archived", "cleanup_failed"]), sql`${executionWorkspaces.metadata}->'gitCacheExpiration'->>'completed' = 'false'`) : undefined, + opts.nestedCloneMigrationEnabled ? and(inArray(executionWorkspaces.status, ["archived", "cleanup_failed"]), + sql`(${executionWorkspaces.metadata}->'nestedCloneMigrationPending') IS NOT NULL`) : undefined, opts.gitCacheExpirationEnabled && opts.isolatedSourceRehomeEnabled ? and(inArray(executionWorkspaces.status, ["archived", "cleanup_failed"]), sql`(${executionWorkspaces.metadata}->'isolatedRepositoryRehome') IS NOT NULL`, sql`(${executionWorkspaces.metadata}->'isolatedRepositoryRehome'->>'cleanedAt') IS NULL`) : undefined), @@ -3271,6 +3281,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic terminalSweepBoundary = now(); terminalWorkspaceDeadlines.clear(); isolatedRehomeDeadlines.clear(); + nestedCloneMigrationDeadlines.clear(); terminalSweepRescanRequested = false; } const boundary = terminalSweepBoundary; @@ -3350,6 +3361,50 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic } if (isClosedExecutionWorkspaceStatus(workspace.status) && !hasPendingGitExpiration(workspace)) continue; } + if (opts.nestedCloneMigrationEnabled && workspace.providerType !== "plugin" + && (isNestedCloneMigrationCandidate(workspace) || hasPendingNestedCloneMigration(workspace))) { + const nestedOnly = isClosedExecutionWorkspaceStatus(workspace.status) && !hasPendingGitExpiration(workspace) + && !hasPendingIsolatedRepositoryRehomeCleanup(workspace); + const notBefore = nestedCloneMigrationNotBefore.get(workspace.id) ?? 0; + if (notBefore > now().getTime()) { + const retryAt = nestedCloneMigrationRetryAt.get(workspace.id); + if (retryAt !== undefined) nestedCloneMigrationDeadlines.set(workspace.id, retryAt); + } else { + const listed = await listNestedCloneMigrationNames(db, workspace).catch(() => [] as string[]); + const batch = nextNestedCloneMigrationBatch(listed, pendingNestedCloneMigrationNames(workspace), + nestedCloneMigrationRotations.get(workspace.id)); + const names = batch.names; + if (nestedCloneMigrationRotations.size >= 4096) nestedCloneMigrationRotations.delete(nestedCloneMigrationRotations.keys().next().value!); + nestedCloneMigrationRotations.set(workspace.id, batch.rotation); + const dryRun = opts.nestedCloneMigrationDryRun ?? false; + let remaining = batch.overflow, retry = batch.retryOverflow && !dryRun; + for (const name of names) { + const migration = await migrateNestedRepositoryClone(db, workspace, name, { dryRun, signal }); + logger.info({ event: "execution_workspace.nested_clone_migration", executionWorkspaceId: workspace.id, + companyId: workspace.companyId, dryRun, ...migration }, "Nested repository clone migration inspected"); + if (migration.outcome !== "migrated" && migration.outcome !== "recovered") remaining = true; + else nestedCloneMigrationRotations.set(workspace.id, { ...batch.rotation, exhausted: null }); + if (!dryRun && migration.reason && NESTED_CLONE_MIGRATION_RETRY_REASONS.has(migration.reason)) retry = true; + } + const [after] = names.length > 0 ? await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspace.id)) : [workspace]; + if (!dryRun && after && hasPendingNestedCloneMigration(after)) retry = true; + if (remaining) { + if (nestedCloneMigrationNotBefore.size >= 4096) nestedCloneMigrationNotBefore.delete(nestedCloneMigrationNotBefore.keys().next().value!); + nestedCloneMigrationNotBefore.set(workspace.id, now().getTime() + ISOLATED_REHOME_RETRY_MS); + } else nestedCloneMigrationNotBefore.delete(workspace.id); + if (retry) { + if (nestedCloneMigrationRetryAt.size >= 4096) nestedCloneMigrationRetryAt.delete(nestedCloneMigrationRetryAt.keys().next().value!); + nestedCloneMigrationRetryAt.set(workspace.id, now().getTime() + ISOLATED_REHOME_RETRY_MS); + nestedCloneMigrationDeadlines.set(workspace.id, now().getTime() + ISOLATED_REHOME_RETRY_MS); + } else { + nestedCloneMigrationRetryAt.delete(workspace.id); + nestedCloneMigrationDeadlines.delete(workspace.id); + } + if (!after) continue; + workspace = after; + } + if (nestedOnly) continue; + } const issueTree = await listWorkspaceIssueTree(workspace); const sourceIssue = issueTree.find((issue) => issue.id === workspace.sourceIssueId); const reopenPending = metadataHasReopenPendingConsumption( @@ -3785,8 +3840,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic }, nextTerminalWorkspaceSweepAt: async () => { - if (terminalWorkspaceDeadlines.size === 0 && isolatedRehomeDeadlines.size === 0) return null; - const next = [...terminalWorkspaceDeadlines.values(), ...isolatedRehomeDeadlines.values()].reduce((earliest, deadline) => Math.min(earliest, deadline), Infinity); + if (terminalWorkspaceDeadlines.size === 0 && isolatedRehomeDeadlines.size === 0 && nestedCloneMigrationDeadlines.size === 0) return null; + const next = [...terminalWorkspaceDeadlines.values(), ...isolatedRehomeDeadlines.values(), ...nestedCloneMigrationDeadlines.values()].reduce((earliest, deadline) => Math.min(earliest, deadline), Infinity); const nowMs = now().getTime(); if (next <= nowMs && opts.terminalWorkspaceInspectionBackoffMs) { return Math.min(nowMs + opts.terminalWorkspaceInspectionBackoffMs, diff --git a/server/src/services/isolated-repository-rehome.ts b/server/src/services/isolated-repository-rehome.ts index 233ca303d88..1b342e52fbe 100644 --- a/server/src/services/isolated-repository-rehome.ts +++ b/server/src/services/isolated-repository-rehome.ts @@ -81,7 +81,7 @@ async function exists(target: string) { return fs.lstat(target).then(() => true, (error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return false; throw error; }); } -async function presentCommits(cwd: string, commits: Iterable, signal?: AbortSignal) { +export async function presentCommits(cwd: string, commits: Iterable, signal?: AbortSignal) { const unique = [...new Set(commits)].filter(sha => SHA.test(sha)); if (unique.length === 0) return new Set(); const output = await git(cwd, ["cat-file", "--batch-check=%(objectname) %(objecttype)"], { signal, input: unique.join("\n") + "\n" }); @@ -105,7 +105,7 @@ function rehomeLockLocations(clone: string, admins: string[], branch: string | n return locations; } -async function rehomeEligible(tx: Db | Tx, workspace: Workspace) { +export async function workspaceMoveEligible(tx: Db | Tx, workspace: Workspace) { if (workspace.metadata?.reopenPendingConsumption === true) return false; const rows = await tx.execute<{ allowed: boolean }>(sql`${expirationScope(workspace)} SELECT ( @@ -154,7 +154,7 @@ async function targetWorktree(db: Db | Tx, workspace: Workspace, base: string, b return target; } -async function assertSafeBaseConfig(base: string, signal?: AbortSignal) { +export async function assertSafeBaseConfig(base: string, signal?: AbortSignal) { const keys = (await git(base, ["config", "--local", "--name-only", "--list"], { signal })).split("\n").map(key => key.trim().toLowerCase()).filter(Boolean); if (keys.some(key => UNSAFE_BASE_CONFIG.test(key))) throw new Error("unsupported_base_git_config"); if (await exists(path.join(base, ".git", "info", "attributes"))) throw new Error("unsupported_base_git_config"); @@ -190,7 +190,7 @@ async function inspectWorktree(workspace: Workspace, clone: string, signal?: Abo return { cwd: real, admin, head: (await git(real, ["rev-parse", "--verify", "HEAD^{commit}"], { signal })).trim() }; } -async function transferPlan(cwd: string, signal?: AbortSignal) { +export async function transferPlan(cwd: string, signal?: AbortSignal) { const regenerable = await findRegenerableWorkspaceDirs(cwd, { signal }); const nestedRoot = path.join(cwd, PROJECT_REPOSITORIES_DIR); const nestedStat = await fs.lstat(nestedRoot).catch(() => null); @@ -209,7 +209,7 @@ async function transferPlan(cwd: string, signal?: AbortSignal) { return plan.filter(item => !plan.some(other => other !== item && item.startsWith(`${other}/`))); } -async function boundary(cwd: string, base: string, head: string, signal?: AbortSignal) { +export async function sharedHistoryBoundary(cwd: string, base: string, head: string, signal?: AbortSignal) { const remoteTips = async (repo: string) => shas(await git(repo, ["for-each-ref", "--format=%(objectname)", "--sort=-committerdate", "refs/remotes"], { signal })); const candidates = [head, ...await remoteTips(base), ...await remoteTips(cwd)]; const inBase = await presentCommits(base, candidates, signal); @@ -217,11 +217,11 @@ async function boundary(cwd: string, base: string, head: string, signal?: AbortS return [...new Set(candidates)].filter(sha => inBase.has(sha) && inClone.has(sha)).slice(0, MAX_BOUNDARY_COMMITS); } -function snapshotKey(snapshot: RestorePointSnapshot) { +export function snapshotKey(snapshot: RestorePointSnapshot) { return JSON.stringify([snapshot.head, snapshot.index, snapshot.worktree, snapshot.ignored, snapshot.modes]); } -async function historyKey(cwd: string, admin: string, signal?: AbortSignal) { +export async function historyKey(cwd: string, admin: string, signal?: AbortSignal) { const reflog = await git(cwd, ["reflog", "show", "--format=%H", "HEAD"], { signal }).catch(() => ""); const original = await fs.readFile(path.join(admin, "ORIG_HEAD"), "utf8").catch(() => ""); return JSON.stringify([shas(reflog), original.trim()]); @@ -249,7 +249,7 @@ async function retargetNestedBranch(checkout: string, from: string) { }); } -async function linkedCheckoutBases(cwd: string, moves: string[]) { +export async function linkedCheckoutBases(cwd: string, moves: string[]) { const bases = new Map(); for (const relative of moves.filter(item => item.startsWith(`${PROJECT_REPOSITORIES_DIR}/`))) { const linked = await readLinkedNestedCheckout(path.join(cwd, relative)).catch(() => { throw new Error("untransferable_managed_checkouts"); }); @@ -284,19 +284,48 @@ async function provenTarget(db: Db | Tx, workspaceId: string, pending: IsolatedR return !other; } +export async function returnTransferItems(original: string, moved: string, moves: string[], conflict: string) { + for (const relative of moves) { + if (await exists(path.join(moved, relative)) && await exists(path.join(original, relative))) throw new Error(conflict); + } + for (const relative of [...moves].reverse()) { + const from = path.join(moved, relative), to = path.join(original, relative); + if (!await exists(from) || await exists(to)) continue; + await fs.mkdir(path.dirname(to), { recursive: true }); + await fs.rename(from, to); + await repairLinkedCheckout(to).catch(() => undefined); + await retargetNestedBranch(to, from).catch(() => undefined); + } +} + +export async function moveTransferItems(from: string, to: string, moves: string[], linkedBases: Map) { + if (moves.some(item => item.startsWith(`${PROJECT_REPOSITORIES_DIR}/`))) await ensureManagedRepositoriesIgnored(to); + for (const relative of moves) { + const destination = path.join(to, relative); + if (await exists(destination)) throw new Error("transfer_conflict"); + await fs.mkdir(path.dirname(destination), { recursive: true }); + await fs.rename(path.join(from, relative), destination); + if (linkedBases.has(relative)) { + await repairLinkedCheckout(destination); + await retargetNestedBranch(destination, path.join(from, relative)); + } + } +} + +export async function assertTransferredItems(from: string, to: string, moves: string[], linkedBases: Map) { + for (const relative of moves) { + if (!await exists(path.join(to, relative)) || await exists(path.join(from, relative))) throw new Error("transfer_incomplete"); + const base = linkedBases.get(relative); + if (!base) continue; + const linked = await readLinkedNestedCheckout(path.join(to, relative)).catch(() => null); + const head = linked ? (await git(linked.checkout, ["symbolic-ref", "--quiet", "--short", "HEAD"]).catch(() => "")).trim() : ""; + if (linked?.base !== base || (head.startsWith("paperclip/") && !head.startsWith(linked.branchPrefix))) throw new Error("transfer_incomplete"); + } +} + async function undoMove(db: Db | Tx, workspaceId: string, pending: IsolatedRepositoryRehomePending, ownsTarget?: boolean) { if (ownsTarget ?? await provenTarget(db, workspaceId, pending)) { - for (const relative of pending.moves) { - if (await exists(path.join(pending.target, relative)) && await exists(path.join(pending.from, relative))) throw new Error("rehome_rollback_conflict"); - } - for (const relative of [...pending.moves].reverse()) { - const moved = path.join(pending.target, relative), original = path.join(pending.from, relative); - if (!await exists(moved) || await exists(original)) continue; - await fs.mkdir(path.dirname(original), { recursive: true }); - await fs.rename(moved, original); - await repairLinkedCheckout(original).catch(() => undefined); - await retargetNestedBranch(original, moved).catch(() => undefined); - } + await returnTransferItems(pending.from, pending.target, pending.moves, "rehome_rollback_conflict"); await git(pending.base, ["worktree", "remove", "--force", "--force", pending.target]).catch(() => undefined); await fs.rm(pending.target, { recursive: true, force: true }); await git(pending.base, ["worktree", "prune"]).catch(() => undefined); @@ -311,7 +340,7 @@ async function undoMove(db: Db | Tx, workspaceId: string, pending: IsolatedRepos } } -function withRehomeLocks(db: Db, workspace: Workspace, work: (tx: Tx) => Promise) { +export function withWorkspaceMoveLocks(db: Db, workspace: Workspace, work: (tx: Tx) => Promise) { return withIsolatedRepositorySourceLock(db, workspace.companyId, workspace.sourceIssueId!, () => withDedicatedDbConnection(db, connection => connection.transaction(async tx => { const rows = await tx.execute<{ locked: boolean; idle_timeout_ms: string | number | null }>(sql`select @@ -327,10 +356,15 @@ function emptyResult(): IsolatedRepositoryRehomeResult { candidateBytes: 0, reclaimedBytes: 0, from: null, to: null }; } -function skip(result: IsolatedRepositoryRehomeResult, error: unknown) { +export function workspaceMoveSkip(error: unknown): { reason: string; failure?: GitExpirationFailure } { const reason = error instanceof IsolatedRepositorySourceBusy ? "source_busy" : error instanceof RestorePointSkip ? error.reason : error instanceof RestorePointRestoreFailure ? error.reason : gitExpirationSkipReason(error); - if (reason === "inspection_failed") result.failure = describeGitExpirationFailure(error); + return reason === "inspection_failed" ? { reason, failure: describeGitExpirationFailure(error) } : { reason }; +} + +function skip(result: IsolatedRepositoryRehomeResult, error: unknown) { + const { reason, failure } = workspaceMoveSkip(error); + if (failure) result.failure = failure; return reason; } @@ -341,11 +375,11 @@ export async function rehomeIsolatedRepositoryWorkspace(db: Db, snapshot: Worksp const candidate = isIsolatedRepositoryRehomeCandidate(snapshot); if (!snapshot.sourceIssueId || (!candidate && !hasPendingIsolatedRepositoryRehomeCleanup(snapshot))) return { ...result, reason: "not_candidate" }; try { - return await withRehomeLocks(db, snapshot, async tx => { + return await withWorkspaceMoveLocks(db, snapshot, async tx => { const [row] = await db.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.id, snapshot.id), eq(executionWorkspaces.companyId, snapshot.companyId))); if (!row || row.updatedAt.getTime() !== snapshot.updatedAt.getTime()) throw new Error("workspace_changed"); if (!candidate) return await cleanupRehomedSource(db, tx, row, result, options); - if (!await rehomeEligible(tx, row)) throw new Error("live_or_unresolved_consumer"); + if (!await workspaceMoveEligible(tx, row)) throw new Error("live_or_unresolved_consumer"); let recipe: GitCacheRestoration | null = null; try { recipe = row.metadata?.[GIT_CACHE_RESTORATION_KEY] === undefined ? null : readGitCacheRestoration(row); } catch { throw new Error("invalid_restoration_recipe"); } @@ -384,7 +418,7 @@ export async function rehomeIsolatedRepositoryWorkspace(db: Db, snapshot: Worksp const before = await currentRestorePointSnapshot(old.cwd, options.signal); const beforeHistory = await historyKey(old.cwd, old.admin, options.signal); if (before.head !== old.head) throw new Error("head_changed"); - const captured = await captureRestorePoint({ cwd: old.cwd, upstreamCommits: await boundary(old.cwd, base, old.head, options.signal), + const captured = await captureRestorePoint({ cwd: old.cwd, upstreamCommits: await sharedHistoryBoundary(old.cwd, base, old.head, options.signal), limits: options.limits, signal: options.signal }); const manifest = captured ? await importRestorePoint({ repositoryCwd: base, payload: captured.payload, signal: options.signal }) : null; const reflog = shas(await git(old.cwd, ["reflog", "show", "--format=%H", "HEAD"], { signal: options.signal }).catch(() => "")); @@ -416,17 +450,7 @@ export async function rehomeIsolatedRepositoryWorkspace(db: Db, snapshot: Worksp await fs.rm(path.join(newAdmin, "logs", "HEAD"), { force: true }); }); if (original) await fs.copyFile(path.join(old.admin, "ORIG_HEAD"), path.join(newAdmin, "ORIG_HEAD")); - if (moves.some(item => item.startsWith(`${PROJECT_REPOSITORIES_DIR}/`))) await ensureManagedRepositoriesIgnored(target); - for (const relative of moves) { - const destination = path.join(target, relative); - if (await exists(destination)) throw new Error("transfer_conflict"); - await fs.mkdir(path.dirname(destination), { recursive: true }); - await fs.rename(path.join(old.cwd, relative), destination); - if (linkedBases.has(relative)) { - await repairLinkedCheckout(destination); - await retargetNestedBranch(destination, path.join(old.cwd, relative)); - } - } + await moveTransferItems(old.cwd, target, moves, linkedBases); await options.hooks?.beforeVerify?.({ from: old.cwd, to: target }); const after = await currentRestorePointSnapshot(old.cwd, options.signal); const moved = await currentRestorePointSnapshot(target, options.signal); @@ -435,19 +459,12 @@ export async function rehomeIsolatedRepositoryWorkspace(db: Db, snapshot: Worksp throw new Error("history_mismatch"); } if ((await git(target, ["symbolic-ref", "--short", "HEAD"], { signal: options.signal })).trim() !== branch) throw new Error("branch_identity_changed"); - for (const relative of moves) { - if (!await exists(path.join(target, relative)) || await exists(path.join(old.cwd, relative))) throw new Error("transfer_incomplete"); - const base = linkedBases.get(relative); - if (!base) continue; - const linked = await readLinkedNestedCheckout(path.join(target, relative)).catch(() => null); - const head = linked ? (await git(linked.checkout, ["symbolic-ref", "--quiet", "--short", "HEAD"]).catch(() => "")).trim() : ""; - if (linked?.base !== base || (head.startsWith("paperclip/") && !head.startsWith(linked.branchPrefix))) throw new Error("transfer_incomplete"); - } + await assertTransferredItems(old.cwd, target, moves, linkedBases); const at = (options.now ?? new Date()).toISOString(); await db.transaction(async updateTx => { const [current] = await updateTx.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, row.id)).for("update"); if (!current || current.updatedAt.getTime() !== row.updatedAt.getTime()) throw new Error("workspace_changed"); - if (!await rehomeEligible(updateTx, current)) throw new Error("live_or_unresolved_consumer"); + if (!await workspaceMoveEligible(updateTx, current)) throw new Error("live_or_unresolved_consumer"); const metadata: Record = { ...current.metadata }; delete metadata.isolatedRemoteSource; delete metadata.gitCacheExpiration; @@ -546,7 +563,7 @@ async function cleanupRehomedSource(db: Db, tx: Tx, row: Workspace, result: Isol result.outcome = "dry_run"; return result; } - if (!await rehomeEligible(tx, row)) return keep("live_or_unresolved_consumer", false); + if (!await workspaceMoveEligible(tx, row)) return keep("live_or_unresolved_consumer", false); if (cloneReal) { await recoverRehomeLocks(clone, row.branchName, row.id); const fromStat = await fs.lstat(from).catch(() => null); diff --git a/server/src/services/nested-clone-migration.ts b/server/src/services/nested-clone-migration.ts new file mode 100644 index 00000000000..eb0e3a87eda --- /dev/null +++ b/server/src/services/nested-clone-migration.ts @@ -0,0 +1,541 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { createHash } from "node:crypto"; +import { execFile as execFileCallback } from "node:child_process"; +import { promisify } from "node:util"; +import { and, eq, sql } from "drizzle-orm"; +import { executionWorkspaces, projectWorkspaces, type Db } from "@paperclipai/db"; +import { PROJECT_REPOSITORIES_DIR } from "@paperclipai/adapter-utils/git-workspace-sync"; +import { acquireGitExpirationLocks, recoverGitExpirationLocks, type GitExpirationLockLocation } from "./git-workspace-expiration-locks.js"; +import { allocatedBytes, withAnonymousGitFallback, type GitExpirationFailure } from "./git-workspace-expiration.js"; +import { GIT_CACHE_RESTORATION_KEY, readGitCacheRestoration, type GitCacheRestoration } from "./git-workspace-restoration.js"; +import { readManagedCheckoutContents, selectRestoredProjectRepositories } from "./git-workspace-contents.js"; +import { applyRestorePoint, captureRestorePoint, currentRestorePointSnapshot, importRestorePoint } from "./git-restore-point.js"; +import { managedProjectCheckoutName } from "./managed-project-checkout-name.js"; +import { createGitRemoteAuthProvider, type GitRemoteAuthProvider } from "./git-credentials.js"; +import { isNestedWorktreeBranchName, isVerifiedNestedRepositoryBase, moveNestedRepositoryWorktree, nestedRepositoryBaseCandidates, nestedWorktreeBranchPrefix, + readLinkedNestedCheckout, removeStaleNestedRegistrations, trackNestedWorktreeUpstream, withNestedRepositoryBaseLock } from "./nested-repository-worktrees.js"; +import { assertSafeBaseConfig, assertTransferredItems, historyKey, linkedCheckoutBases, moveTransferItems, presentCommits, returnTransferItems, + sharedHistoryBoundary, snapshotKey, transferPlan, withWorkspaceMoveLocks, workspaceMoveEligible, workspaceMoveSkip } from "./isolated-repository-rehome.js"; + +const execFile = promisify(execFileCallback); +const SHA = /^[0-9a-f]{40}$/; +const OPEN_STATUSES = ["active", "idle", "in_review"]; +const CHECKOUT_NAME = /^[A-Za-z0-9_-]+-[a-f0-9]{12}$/; +const REPO_ONLY_CWD_SENTINEL = "/__paperclip_repo_only__"; +const STAGING_SUFFIXES = { worktree: ".clone-migrating", aside: ".clone-standalone", trash: ".clone-trash" } as const; +const LOCAL_CONFIG = /^(core\.(repositoryformatversion|filemode|bare|logallrefupdates|ignorecase|precomposeunicode)|remote\.origin\.(url|fetch)|branch\..+\.(remote|merge))$/; +const UNSUPPORTED_GIT_STATE = ["shallow", "MERGE_HEAD", "CHERRY_PICK_HEAD", "REVERT_HEAD", "BISECT_LOG", "rebase-apply", "rebase-merge", "sequencer", + "info/sparse-checkout", "info/attributes", "refs/replace", "modules", "lfs"]; +const FETCH_TIMEOUT_MS = 10 * 60 * 1000; +export const NESTED_CLONE_MIGRATIONS_PER_SWEEP = 8; +export const NESTED_CLONE_MIGRATION_PENDING_KEY = "nestedCloneMigrationPending"; +export const NESTED_CLONE_MIGRATION_RETRY_REASONS: ReadonlySet = new Set(["lifecycle_busy", "source_busy", "live_or_unresolved_consumer", + "worktree_git_busy", "workspace_changed"]); + +type Workspace = typeof executionWorkspaces.$inferSelect; +type PlannedRef = { ref: string; commit: string; upstream: string | null }; +type Paths = { checkout: string; worktree: string; aside: string; trash: string }; +type Source = { repoUrl: string; base: string; candidates: string[]; source: { id: string; repoUrl: string; repoRef: string | null; cwd: string | null } }; +export type NestedCloneMigrationPending = { + base: string; head: string; branch: string | null; refs: PlannedRef[]; moves: string[]; phase: "transfer" | "swap" | "cleanup"; +}; +export type NestedCloneMigrationHooks = { + beforeWorktreeAdd?: (input: { from: string; to: string }) => Promise; + beforeVerify?: (input: { from: string; to: string }) => Promise; + afterAside?: (input: { checkout: string; aside: string }) => Promise; + afterMove?: (input: { checkout: string; aside: string }) => Promise; + afterFinalize?: (input: { checkout: string; aside: string }) => Promise; +}; +export type NestedCloneMigrationResult = { + name: string; outcome: "migrated" | "recovered" | "rolled_back" | "dry_run" | "skipped"; reason: string | null; + branchName: string | null; candidateBytes: number; reclaimedBytes: number; failure?: GitExpirationFailure; +}; +export type NestedCloneMigrationOptions = { + dryRun?: boolean; signal?: AbortSignal; hooks?: NestedCloneMigrationHooks; resolveGitAuth?: GitRemoteAuthProvider; +}; + +async function git(cwd: string, args: string[], options: { signal?: AbortSignal; input?: string; timeout?: number; auth?: Awaited> } = {}) { + const child = execFile("git", [...options.auth?.configArgs ?? [], "-c", "core.fsmonitor=false", "-c", `core.hooksPath=${os.devNull}`, "-C", cwd, ...args], { + timeout: options.timeout ?? 120_000, signal: options.signal, maxBuffer: 256 * 1024 * 1024, + env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_OPTIONAL_LOCKS: "0", ...options.auth?.env }, + }); + if (options.input !== undefined) child.child.stdin?.end(options.input); + return (await child).stdout; +} + +const shas = (output: string) => output.split("\n").map(line => line.trim()).filter(line => SHA.test(line)); + +async function exists(target: string) { + return fs.lstat(target).then(() => true, (error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return false; throw error; }); +} + +async function canonicalPath(target: string) { + return path.join(await fs.realpath(path.dirname(target)), path.basename(target)); +} + +export function isNestedCloneMigrationCandidate(workspace: Workspace) { + return workspace.providerType !== "plugin" && OPEN_STATUSES.includes(workspace.status) && !workspace.closedAt + && Boolean(workspace.cwd && workspace.projectId && workspace.sourceIssueId); +} + +function canRecover(workspace: Workspace, name: string) { + return workspace.providerType !== "plugin" && Boolean(workspace.cwd && workspace.projectId && workspace.sourceIssueId) + && pendingRecords(workspace)[name] !== undefined; +} + +function pendingRecords(workspace: Pick) { + const value = workspace.metadata?.[NESTED_CLONE_MIGRATION_PENDING_KEY]; + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} + +export function pendingNestedCloneMigrationNames(workspace: Pick) { + return Object.keys(pendingRecords(workspace)).filter(name => CHECKOUT_NAME.test(name)); +} + +export type NestedCloneMigrationRotation = { visited: ReadonlySet; exhausted: string | null }; + +export function nextNestedCloneMigrationBatch(listed: string[], pending: string[], rotation: NestedCloneMigrationRotation | undefined, + limit = NESTED_CLONE_MIGRATIONS_PER_SWEEP) { + const listing = listed.join("\n"); + const visited = rotation?.visited ?? new Set(); + const first = pending.filter(name => listed.includes(name)); + const rest = listed.filter(name => !first.includes(name)); + const names = [...first, ...rest.filter(name => !visited.has(name)), ...rest.filter(name => visited.has(name))].slice(0, limit); + const seen = new Set([...[...visited].filter(name => listed.includes(name)), ...names]); + const cycleOpen = listed.some(name => !seen.has(name)); + const overflow = listed.length > names.length; + const exhausted = rotation?.exhausted === listing ? listing : !cycleOpen && overflow ? listing : null; + return { names, overflow, retryOverflow: cycleOpen && rotation?.exhausted !== listing, + rotation: { visited: cycleOpen ? seen : new Set(), exhausted } satisfies NestedCloneMigrationRotation }; +} + +export function hasPendingNestedCloneMigration(workspace: Pick) { + return Object.keys(pendingRecords(workspace)).length > 0; +} + +function readPending(workspace: Workspace, name: string, prefix: string): NestedCloneMigrationPending | null { + const value = pendingRecords(workspace)[name] as NestedCloneMigrationPending | undefined; + if (value === undefined) return null; + const safeRelative = (item: unknown) => typeof item === "string" && item.length > 0 && !path.isAbsolute(item) && !item.split("/").includes(".."); + if (!value || typeof value !== "object" || typeof value.base !== "string" || !path.isAbsolute(value.base) || !SHA.test(value.head) + || !(value.branch === null || typeof value.branch === "string") || !["transfer", "swap", "cleanup"].includes(value.phase) + || !Array.isArray(value.moves) || !value.moves.every(safeRelative) || !Array.isArray(value.refs) + || !value.refs.every(ref => ref && typeof ref.ref === "string" && ref.ref.startsWith(`refs/heads/${prefix}`) && SHA.test(ref.commit) + && (ref.upstream === null || typeof ref.upstream === "string")) + || (value.branch !== null && !value.refs.some(ref => ref.ref === `refs/heads/${value.branch}`))) throw new Error("invalid_nested_migration_record"); + return value; +} + +async function writePending(db: Db, workspaceId: string, name: string, value: NestedCloneMigrationPending | null) { + const key = NESTED_CLONE_MIGRATION_PENDING_KEY; + await db.update(executionWorkspaces).set({ metadata: value + ? sql`jsonb_set(coalesce(${executionWorkspaces.metadata}, '{}'::jsonb), ${`{${key}}`}::text[], + coalesce(${executionWorkspaces.metadata}->${key}, '{}'::jsonb) || jsonb_build_object(${name}::text, ${JSON.stringify(value)}::jsonb))` + : sql`CASE WHEN (${executionWorkspaces.metadata}->${key}) - ${name}::text = '{}'::jsonb THEN ${executionWorkspaces.metadata} - ${key}::text + ELSE ${executionWorkspaces.metadata} #- ${`{${key},${name}}`}::text[] END` }).where(eq(executionWorkspaces.id, workspaceId)); +} + +export async function listNestedCloneMigrationNames(db: Db, workspace: Workspace) { + const names = new Set(Object.keys(pendingRecords(workspace)).filter(name => CHECKOUT_NAME.test(name))); + if (!workspace.cwd || !isNestedCloneMigrationCandidate(workspace)) return [...names].sort(); + const root = path.join(workspace.cwd, PROJECT_REPOSITORIES_DIR); + const stat = await fs.lstat(root).catch(() => null); + if (!stat?.isDirectory() || stat.isSymbolicLink()) return [...names].sort(); + const rows = await db.select({ id: projectWorkspaces.id, repoUrl: projectWorkspaces.repoUrl, repoRef: projectWorkspaces.repoRef, cwd: projectWorkspaces.cwd }) + .from(projectWorkspaces).where(and(eq(projectWorkspaces.companyId, workspace.companyId), eq(projectWorkspaces.projectId, workspace.projectId!))); + let selected = rows; + try { + selected = selectRestoredProjectRepositories(rows, workspace.metadata?.[GIT_CACHE_RESTORATION_KEY] === undefined ? undefined : readGitCacheRestoration(workspace)?.contents); + } catch { selected = rows; } + const configured = new Set(selected.filter(row => row.repoUrl) + .map(row => managedProjectCheckoutName({ id: row.id, repoUrl: row.repoUrl!, repoRef: row.repoRef, cwd: row.cwd }))); + for (const name of configured) { + if ((await Promise.all(Object.values(STAGING_SUFFIXES).map(suffix => exists(path.join(root, `${name}${suffix}`))))).some(Boolean)) { names.add(name); continue; } + const marker = await fs.lstat(path.join(root, name, ".git")).catch(() => null); + if (marker?.isDirectory() && !marker.isSymbolicLink()) names.add(name); + } + return [...names].sort(); +} + +async function nestedCloneSource(db: Db, workspace: Workspace, name: string): Promise { + const rows = await db.select({ id: projectWorkspaces.id, repoUrl: projectWorkspaces.repoUrl, repoRef: projectWorkspaces.repoRef, cwd: projectWorkspaces.cwd }) + .from(projectWorkspaces).where(and(eq(projectWorkspaces.companyId, workspace.companyId), eq(projectWorkspaces.projectId, workspace.projectId!))); + let selected: typeof rows; + try { + const contents = workspace.metadata?.[GIT_CACHE_RESTORATION_KEY] === undefined ? undefined : readGitCacheRestoration(workspace)?.contents; + selected = selectRestoredProjectRepositories(rows, contents); + } catch { throw new Error("invalid_restoration_recipe"); } + const source = selected.find(row => row.repoUrl && managedProjectCheckoutName({ id: row.id, repoUrl: row.repoUrl, repoRef: row.repoRef, cwd: row.cwd }) === name); + if (!source?.repoUrl) throw new Error("unknown_managed_checkout"); + if (source.cwd && source.cwd !== REPO_ONLY_CWD_SENTINEL && await fs.stat(source.cwd).then(entry => entry.isDirectory(), () => false)) { + throw new Error("local_source_checkout"); + } + const candidates = nestedRepositoryBaseCandidates({ companyId: workspace.companyId, projectId: workspace.projectId!, repoUrl: source.repoUrl }); + let mismatch = false; + for (const candidate of candidates) { + if (await isVerifiedNestedRepositoryBase(candidate, candidates, source.repoUrl)) { + return { repoUrl: source.repoUrl, base: await fs.realpath(candidate), candidates, source: { ...source, repoUrl: source.repoUrl } }; + } + if (await exists(path.join(candidate, ".git"))) mismatch = true; + } + throw new Error(mismatch ? "shared_base_repo_mismatch" : "no_shared_base"); +} + +async function excludeLines(gitDir: string) { + return (await fs.readFile(path.join(gitDir, "info", "exclude"), "utf8").catch(() => "")).split(/\r?\n/).map(line => line.trim()) + .filter(line => line && !line.startsWith("#")); +} + +async function inspectStandaloneClone(checkout: string, repoUrl: string, base: string, signal?: AbortSignal) { + const stat = await fs.lstat(checkout).catch(() => null); + if (!stat) throw new Error("missing_managed_checkout"); + if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("unproven_managed_checkout"); + const marker = await fs.lstat(path.join(checkout, ".git")).catch(() => null); + if (marker?.isFile()) throw new Error("already_worktree_backed"); + if (!marker?.isDirectory() || marker.isSymbolicLink()) throw new Error("unproven_managed_checkout"); + const real = await fs.realpath(checkout); + if (real !== await canonicalPath(checkout) || await fs.realpath(path.join(checkout, ".git")) !== path.join(real, ".git")) throw new Error("unproven_managed_checkout"); + if ((await git(real, ["rev-parse", "--show-toplevel"], { signal })).trim() !== real + || (await git(real, ["rev-parse", "--path-format=absolute", "--git-common-dir"], { signal })).trim() !== path.join(real, ".git")) { + throw new Error("unproven_managed_checkout"); + } + if (await git(real, ["config", "--local", "--null", "--get-all", "remote.origin.url"], { signal }).catch(() => "") !== `${repoUrl}\0`) throw new Error("foreign_origin"); + const keys = (await git(real, ["config", "--local", "--name-only", "--list"], { signal })).split("\n").map(key => key.trim()).filter(Boolean); + if (keys.some(key => !LOCAL_CONFIG.test(key))) throw new Error("unsupported_local_git_config"); + if ((await git(real, ["worktree", "list", "--porcelain"], { signal })).split("\n").filter(line => line.startsWith("worktree ")).length !== 1) { + throw new Error("nested_clone_has_worktrees"); + } + for (const name of UNSUPPORTED_GIT_STATE) if (await exists(path.join(real, ".git", name))) throw new Error("unsupported_local_git_state"); + const hooks = await fs.readdir(path.join(real, ".git", "hooks")).catch(() => [] as string[]); + if (hooks.some(hook => !hook.endsWith(".sample"))) throw new Error("nested_clone_has_hooks"); + const shared = new Set(await excludeLines(path.join(base, ".git"))); + if ((await excludeLines(path.join(real, ".git"))).some(line => !shared.has(line))) throw new Error("unsupported_info_exclude"); + if ((await git(real, ["for-each-ref", "--format=%(refname)", "refs/stash"], { signal })).trim()) throw new Error("nested_clone_has_stash"); + const refs = (await git(real, ["for-each-ref", "--format=%(refname)"], { signal })).split("\n").filter(Boolean); + if (refs.some(ref => !/^refs\/(heads|tags|remotes\/origin)\//.test(ref))) throw new Error("unsupported_nested_clone_refs"); + if ((await git(real, ["ls-files", "-v"], { signal })).split("\n").some(line => /^[a-zS] /.test(line))) throw new Error("unsupported_index_flags"); + const head = (await git(real, ["rev-parse", "--verify", "HEAD^{commit}"], { signal })).trim(); + const branch = (await git(real, ["symbolic-ref", "--quiet", "--short", "HEAD"], { signal }).catch(() => "")).trim() || null; + return { checkout: real, admin: path.join(real, ".git"), head, branch }; +} + +async function onRemote(base: string, commit: string, signal?: AbortSignal) { + const missing = await git(base, ["rev-list", "--max-count=1", commit, "--not", "--remotes=origin"], { signal }).catch(() => null); + return missing !== null && missing.trim() === ""; +} + +async function upstreamOf(clone: string, branch: string) { + const remote = (await git(clone, ["config", "--local", "--get", `branch.${branch}.remote`]).catch(() => "")).trim(); + const merge = (await git(clone, ["config", "--local", "--get", `branch.${branch}.merge`]).catch(() => "")).trim(); + return remote === "origin" && merge.startsWith("refs/heads/") ? merge.slice("refs/heads/".length) : branch; +} + +async function planRefs(clone: { checkout: string; head: string; branch: string | null }, base: string, prefix: string, signal?: AbortSignal) { + const planned: PlannedRef[] = []; + if (clone.branch) planned.push({ ref: `refs/heads/${prefix}${clone.branch}`, commit: clone.head, upstream: await upstreamOf(clone.checkout, clone.branch) }); + const refs = (await git(clone.checkout, ["for-each-ref", "--format=%(objectname) %(objecttype) %(*objectname) %(*objecttype) %(refname)", "refs/heads", "refs/tags", "refs/remotes/origin"], { signal })) + .split("\n").filter(Boolean).map(line => line.split(" ")).filter(([, , , , ref]) => ref !== "refs/remotes/origin/HEAD"); + for (const [sha, type, peeled, peeledType, ref] of refs) { + if (!ref || ref === `refs/heads/${clone.branch}`) continue; + const commit = type === "commit" ? sha! : peeledType === "commit" ? peeled! : null; + if (!commit || !SHA.test(commit)) throw new Error("unsupported_nested_clone_tag"); + if (await onRemote(base, commit, signal)) continue; + const name = ref.startsWith("refs/heads/") ? ref.slice("refs/heads/".length) + : ref.startsWith("refs/tags/") ? `tags/${ref.slice("refs/tags/".length)}` : `remotes/${ref.slice("refs/remotes/".length)}`; + planned.push({ ref: `refs/heads/${prefix}${name}`, commit, upstream: ref.startsWith("refs/heads/") ? await upstreamOf(clone.checkout, name) : null }); + } + const names = new Set(); + for (const { ref } of planned) { + await git(base, ["check-ref-format", ref]).catch(() => { throw new Error("unsupported_branch_name"); }); + if (names.has(ref) || (await git(base, ["rev-parse", "--verify", "-q", ref]).catch(() => "")).trim()) throw new Error("nested_clone_ref_conflict"); + names.add(ref); + } + return planned; +} + +async function assertNoBranchOnlyReflogHistory(clone: { checkout: string; admin: string; head: string }, signal?: AbortSignal) { + const local = (await git(clone.checkout, ["reflog", "show", "--all", "--format=%H %gD"], { signal }).catch(() => "")) + .split("\n").map(line => line.trim().split(" ")).filter(([sha, selector]) => SHA.test(sha ?? "") && /^refs\/(heads|tags)\//.test(selector ?? "")).map(([sha]) => sha!); + if (local.length === 0) return; + const covered = [clone.head, ...shas(await git(clone.checkout, ["reflog", "show", "--format=%H", "HEAD"], { signal }).catch(() => "")), + ...shas((await fs.readFile(path.join(clone.admin, "ORIG_HEAD"), "utf8").catch(() => ""))), + ...shas(await git(clone.checkout, ["for-each-ref", "--format=%(objectname)", "refs/heads"], { signal })), + ...shas(await git(clone.checkout, ["for-each-ref", "--format=%(*objectname)", "refs/tags"], { signal })), + ...shas(await git(clone.checkout, ["for-each-ref", "--format=%(objectname)", "refs/tags"], { signal }))]; + const missing = await git(clone.checkout, ["rev-list", "--max-count=1", "--stdin"], { signal, + input: [...new Set(local)].join("\n") + "\n" + [...new Set(covered)].map(sha => `^${sha}`).join("\n") + "\n--not\n--remotes\n" }); + if (missing.trim()) throw new Error("unique_branch_reflog_commits"); +} + +async function preservedCandidates(checkout: string, head: string, boundary: string[], planned: PlannedRef[], signal?: AbortSignal) { + const candidates = [...new Set(planned.map(ref => ref.commit).filter(commit => commit !== head))]; + if (candidates.length === 0) return []; + const unreachable = new Set(shas(await git(checkout, ["rev-list", "--stdin"], { signal, + input: [...candidates, `^${head}`, ...boundary.map(commit => `^${commit}`)].join("\n") + "\n" }))); + return candidates.filter(commit => unreachable.has(commit)); +} + +function lockToken(workspaceId: string, name: string) { + return createHash("sha256").update(`paperclip-nested-clone-migration:${workspaceId}:${name}`).digest("hex"); +} + +function cloneLockLocations(checkout: string, branch: string | null): GitExpirationLockLocation[] { + return [...["index.lock", "HEAD.lock"].map(name => ({ root: checkout, relativePath: path.join(".git", name), checkout: null })), + ...(branch ? [{ root: checkout, relativePath: path.join(".git", "refs", "heads", `${branch}.lock`), checkout: null }] : [])]; +} + +async function fetchBase(base: string, repoUrl: string, resolveGitAuth: GitRemoteAuthProvider, signal?: AbortSignal) { + const auth = await resolveGitAuth(repoUrl).catch(() => null); + await git(base, ["fetch", "--quiet", "--prune", "--no-write-fetch-head", "origin"], { auth, signal, timeout: FETCH_TIMEOUT_MS }).catch(() => undefined); +} + +async function deletePlannedRefs(base: string, refs: PlannedRef[]) { + const held = (await git(base, ["worktree", "list", "--porcelain"])).split("\n").filter(line => line.startsWith("branch ")).map(line => line.slice(7)); + for (const { ref, commit } of [...refs].reverse()) { + if (held.includes(ref) || (await git(base, ["rev-parse", "--verify", "-q", ref]).catch(() => "")).trim() !== commit) continue; + await git(base, ["update-ref", "-d", ref, commit]); + const branch = ref.slice("refs/heads/".length); + for (const key of ["remote", "merge"]) await git(base, ["config", "--local", "--unset-all", `branch.${branch}.${key}`]).catch(() => undefined); + } +} + +async function rollBack(base: string, paths: Paths, pending: NestedCloneMigrationPending) { + if (await exists(paths.worktree)) { + const linked = await readLinkedNestedCheckout(paths.worktree).catch(() => null); + if (!linked || linked.base !== base) throw new Error("nested_migration_recovery_conflict"); + await returnTransferItems(paths.checkout, paths.worktree, pending.moves, "nested_migration_rollback_conflict"); + await git(base, ["worktree", "remove", "--force", "--force", paths.worktree], { timeout: FETCH_TIMEOUT_MS }); + } + await removeStaleNestedRegistrations(base, paths.worktree); + await deletePlannedRefs(base, pending.refs); +} + +async function forgetRemovedWorkspace(db: Db, row: Workspace, name: string) { + const value = pendingRecords(row)[name] as NestedCloneMigrationPending | undefined; + const base = value && typeof value.base === "string" ? value.base : null; + if (base && Array.isArray(value!.refs) && await exists(path.join(base, ".git"))) { + await withNestedRepositoryBaseLock(base, async () => { + const root = path.join(row.cwd!, PROJECT_REPOSITORIES_DIR); + for (const target of [path.join(root, name), path.join(root, `${name}${STAGING_SUFFIXES.worktree}`)]) { + await removeStaleNestedRegistrations(base, target).catch(() => undefined); + } + const pushed: PlannedRef[] = []; + for (const ref of value!.refs) { + if (ref && typeof ref.ref === "string" && ref.ref.startsWith("refs/heads/") && isNestedWorktreeBranchName(ref.ref.slice("refs/heads/".length)) + && SHA.test(ref.commit) && await onRemote(base, ref.commit)) pushed.push(ref); + } + await deletePlannedRefs(base, pushed); + }); + } + await writePending(db, row.id, name, null); +} + +function migratedRecipe(row: Workspace, name: string, contents: Awaited>): GitCacheRestoration | undefined { + if (row.metadata?.[GIT_CACHE_RESTORATION_KEY] === undefined) return undefined; + try { + const recipe = readGitCacheRestoration(row); + if (!recipe?.contents?.checkouts.some(checkout => checkout.name === name)) return undefined; + const next: GitCacheRestoration = { ...recipe, contents: { ...recipe.contents, checkouts: recipe.contents.checkouts.map(checkout => checkout.name === name + ? { ...checkout, commit: contents.commit, branchName: contents.branchName, worktree: contents.worktree } : checkout) } }; + readGitCacheRestoration({ ...row, metadata: { ...row.metadata, [GIT_CACHE_RESTORATION_KEY]: next } }); + return next; + } catch { return undefined; } +} + +async function finalize(db: Db, row: Workspace, name: string, paths: Paths, source: Source, pending: NestedCloneMigrationPending, + hooks?: NestedCloneMigrationHooks) { + if (pending.phase !== "cleanup") { + const linked = await readLinkedNestedCheckout(paths.checkout, source.candidates).catch(() => null); + if (!linked || linked.base !== source.base || linked.checkout !== paths.checkout) throw new Error("nested_migration_recovery_conflict"); + const contents = await readManagedCheckoutContents(paths.checkout, name, source.source); + await db.transaction(async tx => { + const [current] = await tx.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, row.id)).for("update"); + if (!current) throw new Error("workspace_changed"); + const recipe = migratedRecipe(current, name, contents); + const records = { ...pendingRecords(current), [name]: { ...pending, phase: "cleanup" } }; + await tx.update(executionWorkspaces).set({ metadata: { ...current.metadata, [NESTED_CLONE_MIGRATION_PENDING_KEY]: records, + ...(recipe ? { [GIT_CACHE_RESTORATION_KEY]: recipe } : {}) } }).where(eq(executionWorkspaces.id, row.id)); + }); + await hooks?.afterFinalize?.({ checkout: paths.checkout, aside: paths.aside }); + } + let reclaimed = 0; + if (await exists(paths.aside)) { + reclaimed += await allocatedBytes(paths.aside, () => true).catch(() => 0); + if (await exists(paths.trash)) await fs.rm(paths.trash, { recursive: true, force: true }); + await fs.rename(paths.aside, paths.trash); + } + if (await exists(paths.trash)) { + if (!reclaimed) reclaimed += await allocatedBytes(paths.trash, () => true).catch(() => 0); + await fs.rm(paths.trash, { recursive: true, force: true }); + } + await writePending(db, row.id, name, null); + return reclaimed; +} + +async function settle(db: Db, row: Workspace, name: string, paths: Paths, source: Source, pending: NestedCloneMigrationPending, + hooks?: NestedCloneMigrationHooks): Promise<{ forward: boolean; reclaimed: number }> { + if (pending.phase === "cleanup") return { forward: true, reclaimed: await finalize(db, row, name, paths, source, pending, hooks) }; + if (pending.phase === "swap" && await exists(paths.aside)) { + if (!await exists(paths.checkout)) { + const staged = await readLinkedNestedCheckout(paths.worktree).catch(() => null); + if (!staged || staged.base !== source.base) throw new Error("nested_migration_recovery_conflict"); + await moveNestedRepositoryWorktree(source.base, paths.worktree, paths.checkout); + } else if (await exists(paths.worktree)) throw new Error("nested_migration_recovery_conflict"); + return { forward: true, reclaimed: await finalize(db, row, name, paths, source, pending, hooks) }; + } + if (await exists(paths.aside)) throw new Error("nested_migration_recovery_conflict"); + const marker = await fs.lstat(path.join(paths.checkout, ".git")).catch(() => null); + if (!marker?.isDirectory() || marker.isSymbolicLink()) throw new Error("nested_migration_recovery_conflict"); + await rollBack(source.base, paths, pending); + await writePending(db, row.id, name, null); + return { forward: false, reclaimed: 0 }; +} + +export async function migrateNestedRepositoryClone(db: Db, snapshot: Workspace, name: string, + options: NestedCloneMigrationOptions = {}): Promise { + const result: NestedCloneMigrationResult = { name, outcome: "skipped", reason: null, branchName: null, candidateBytes: 0, reclaimedBytes: 0 }; + if ((!isNestedCloneMigrationCandidate(snapshot) && !canRecover(snapshot, name)) || !CHECKOUT_NAME.test(name)) return { ...result, reason: "not_candidate" }; + const { signal, hooks } = options; + try { + return await withWorkspaceMoveLocks(db, snapshot, async tx => { + let [row] = await db.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.id, snapshot.id), eq(executionWorkspaces.companyId, snapshot.companyId))); + if (!row || (!isNestedCloneMigrationCandidate(row) && !canRecover(row, name)) || row.cwd !== snapshot.cwd) throw new Error("workspace_changed"); + if (!isNestedCloneMigrationCandidate(row) && !await exists(row.cwd!)) { + if (options.dryRun) return { ...result, outcome: "dry_run" as const, reason: "interrupted_migration_pending" }; + await forgetRemovedWorkspace(db, row, name); + return { ...result, outcome: "rolled_back" as const, reason: "workspace_removed" }; + } + const cwd = await fs.realpath(row.cwd!); + const rootStat = await fs.lstat(path.join(cwd, PROJECT_REPOSITORIES_DIR)).catch(() => null); + if (!rootStat?.isDirectory() || rootStat.isSymbolicLink()) throw new Error("missing_managed_checkout"); + const root = path.join(cwd, PROJECT_REPOSITORIES_DIR); + const paths: Paths = { checkout: path.join(root, name), worktree: path.join(root, `${name}${STAGING_SUFFIXES.worktree}`), + aside: path.join(root, `${name}${STAGING_SUFFIXES.aside}`), trash: path.join(root, `${name}${STAGING_SUFFIXES.trash}`) }; + const prefix = await nestedWorktreeBranchPrefix(paths.checkout); + const source = await nestedCloneSource(db, row, name); + const token = lockToken(row.id, name); + const interrupted = readPending(row, name, prefix); + if (interrupted) { + if (interrupted.base !== source.base) throw new Error("invalid_nested_migration_record"); + if (options.dryRun) return { ...result, outcome: "dry_run" as const, reason: "interrupted_migration_pending", branchName: interrupted.branch }; + const settled = await withNestedRepositoryBaseLock(source.base, async () => { + await recoverGitExpirationLocks(cloneLockLocations(paths.checkout, interrupted.branch?.slice(prefix.length) ?? null), token).catch(() => undefined); + return settle(db, row!, name, paths, source, interrupted, hooks); + }); + if (settled.forward) return { ...result, outcome: "recovered" as const, branchName: interrupted.branch, reclaimedBytes: settled.reclaimed }; + result.outcome = "rolled_back"; + [row] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, snapshot.id)); + if (!row) throw new Error("workspace_changed"); + if (!isNestedCloneMigrationCandidate(row)) return result; + } + if (await exists(paths.trash) && !options.dryRun) await fs.rm(paths.trash, { recursive: true, force: true }); + if (await exists(paths.aside) || await exists(paths.worktree)) throw new Error("nested_migration_staging_exists"); + if (!await workspaceMoveEligible(tx, row)) throw new Error("live_or_unresolved_consumer"); + const inspected = await inspectStandaloneClone(paths.checkout, source.repoUrl, source.base, signal); + await recoverGitExpirationLocks(cloneLockLocations(inspected.checkout, inspected.branch), token).catch(() => undefined); + result.branchName = inspected.branch ? `${prefix}${inspected.branch}` : null; + result.candidateBytes = await allocatedBytes(inspected.admin, () => { signal?.throwIfAborted(); return true; }).catch(() => 0); + if (options.dryRun) return { ...result, outcome: "dry_run" as const }; + return await withNestedRepositoryBaseLock(source.base, async () => { + const base = source.base; + await assertSafeBaseConfig(base, signal); + await fetchBase(base, source.repoUrl, options.resolveGitAuth + ?? withAnonymousGitFallback(createGitRemoteAuthProvider(db, row!.companyId, { issueId: row!.sourceIssueId! })), signal); + await assertSafeBaseConfig(base, signal); + await assertNoBranchOnlyReflogHistory(inspected, signal); + const planned = await planRefs(inspected, base, prefix, signal); + const moves = await transferPlan(inspected.checkout, signal); + const linkedBases = await linkedCheckoutBases(inspected.checkout, moves); + const locks = await acquireGitExpirationLocks(cloneLockLocations(inspected.checkout, inspected.branch), token) + .catch(() => { throw new Error("worktree_git_busy"); }); + const pending: NestedCloneMigrationPending = { base, head: inspected.head, branch: result.branchName, refs: planned, moves, phase: "transfer" }; + let recorded = false, swapped = false; + try { + const before = await currentRestorePointSnapshot(inspected.checkout, signal); + const beforeHistory = await historyKey(inspected.checkout, inspected.admin, signal); + if (before.head !== inspected.head) throw new Error("head_changed"); + const boundary = await sharedHistoryBoundary(inspected.checkout, base, inspected.head, signal); + const captured = await captureRestorePoint({ cwd: inspected.checkout, upstreamCommits: boundary, signal, + preserveCommits: await preservedCandidates(inspected.checkout, inspected.head, boundary, planned, signal) }); + const manifest = captured ? await importRestorePoint({ repositoryCwd: base, payload: captured.payload, signal }) : null; + const reflog = shas(await git(inspected.checkout, ["reflog", "show", "--format=%H", "HEAD"], { signal }).catch(() => "")); + const original = (await fs.readFile(path.join(inspected.admin, "ORIG_HEAD"), "utf8").catch(() => "")).trim(); + const history = [inspected.head, ...reflog, ...(SHA.test(original) ? [original] : []), ...planned.map(ref => ref.commit)]; + const present = await presentCommits(base, history, signal); + if (history.some(sha => !present.has(sha))) throw new Error("history_not_transferred"); + await writePending(db, row!.id, name, pending); + recorded = true; + for (const ref of planned) { + await git(base, ["update-ref", ref.ref, ref.commit, ""], { signal }); + await trackNestedWorktreeUpstream(base, ref.ref.slice("refs/heads/".length), ref.upstream); + } + await hooks?.beforeWorktreeAdd?.({ from: inspected.checkout, to: paths.worktree }); + await assertSafeBaseConfig(base, signal); + await git(base, ["worktree", "add", "--quiet", ...(result.branchName ? [paths.worktree, result.branchName] : ["--detach", paths.worktree, inspected.head])], + { signal, timeout: FETCH_TIMEOUT_MS }); + const staged = await canonicalPath(paths.worktree); + await assertSafeBaseConfig(base, signal); + if (manifest) await applyRestorePoint({ cwd: staged, manifest, signal }); + const admin = (await git(staged, ["rev-parse", "--absolute-git-dir"], { signal })).trim(); + await fs.mkdir(path.join(admin, "logs"), { recursive: true }); + await fs.copyFile(path.join(inspected.admin, "logs", "HEAD"), path.join(admin, "logs", "HEAD")).catch(async (error: NodeJS.ErrnoException) => { + if (error.code !== "ENOENT") throw error; + await fs.rm(path.join(admin, "logs", "HEAD"), { force: true }); + }); + if (SHA.test(original)) await fs.copyFile(path.join(inspected.admin, "ORIG_HEAD"), path.join(admin, "ORIG_HEAD")); + else await fs.rm(path.join(admin, "ORIG_HEAD"), { force: true }); + await assertSafeBaseConfig(base, signal); + await moveTransferItems(inspected.checkout, staged, moves, linkedBases); + await hooks?.beforeVerify?.({ from: inspected.checkout, to: staged }); + const after = await currentRestorePointSnapshot(inspected.checkout, signal); + const moved = await currentRestorePointSnapshot(staged, signal); + if (snapshotKey(after) !== snapshotKey(before) || snapshotKey(moved) !== snapshotKey(before)) throw new Error("snapshot_mismatch"); + if (await historyKey(inspected.checkout, inspected.admin, signal) !== beforeHistory || await historyKey(staged, admin, signal) !== beforeHistory) { + throw new Error("history_mismatch"); + } + const symbolic = (await git(staged, ["symbolic-ref", "--quiet", "HEAD"], { signal }).catch(() => "")).trim(); + if (symbolic !== (result.branchName ? `refs/heads/${result.branchName}` : "")) throw new Error("branch_identity_changed"); + for (const ref of planned) { + if ((await git(base, ["rev-parse", "--verify", "-q", ref.ref], { signal }).catch(() => "")).trim() !== ref.commit) throw new Error("history_not_transferred"); + } + await assertTransferredItems(inspected.checkout, staged, moves, linkedBases); + const [current] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, row!.id)); + if (!current || current.updatedAt.getTime() !== row!.updatedAt.getTime() || current.cwd !== row!.cwd) throw new Error("workspace_changed"); + if (!await workspaceMoveEligible(tx, current)) throw new Error("live_or_unresolved_consumer"); + pending.phase = "swap"; + await writePending(db, row!.id, name, pending); + await fs.rename(inspected.checkout, paths.aside); + swapped = true; + await hooks?.afterAside?.({ checkout: paths.checkout, aside: paths.aside }); + await moveNestedRepositoryWorktree(base, staged, paths.checkout); + await hooks?.afterMove?.({ checkout: paths.checkout, aside: paths.aside }); + if ((await git(paths.checkout, ["rev-parse", "--verify", "HEAD^{commit}"], { signal })).trim() !== inspected.head) throw new Error("head_changed"); + result.reclaimedBytes = await finalize(db, row!, name, paths, source, pending, hooks); + result.outcome = "migrated"; + result.reason = null; + return result; + } finally { + try { + if (!swapped && recorded) { + await rollBack(base, paths, pending); + await writePending(db, row!.id, name, null); + } + } finally { await locks.release().catch(() => undefined); } + } + }); + }); + } catch (error) { + signal?.throwIfAborted(); + const { reason, failure } = workspaceMoveSkip(error); + if (result.outcome !== "rolled_back") result.outcome = "skipped"; + return { ...result, reason, ...(failure ? { failure } : {}) }; + } +} + +export async function migrateNestedRepositoryClones(db: Db, workspace: Workspace, options: NestedCloneMigrationOptions = {}) { + const results: NestedCloneMigrationResult[] = []; + for (const name of await listNestedCloneMigrationNames(db, workspace)) results.push(await migrateNestedRepositoryClone(db, workspace, name, options)); + return results; +} diff --git a/server/src/services/nested-repository-worktrees.ts b/server/src/services/nested-repository-worktrees.ts index ae48ab2c947..8a2969ec2f3 100644 --- a/server/src/services/nested-repository-worktrees.ts +++ b/server/src/services/nested-repository-worktrees.ts @@ -147,7 +147,7 @@ export async function withNestedRepositoryBaseLock(base: string, action: () = } } -async function removeStaleRegistrations(base: string, checkout: string) { +export async function removeStaleNestedRegistrations(base: string, checkout: string) { const expected = path.join(checkout, ".git"); const root = path.join(base, ".git", "worktrees"); for (const admin of await fs.readdir(root).catch(() => [] as string[])) { @@ -204,7 +204,7 @@ export async function addNestedRepositoryWorktree(input: { const { branch, start } = await resolveStartPoint(input.base, input.repoRef); if (!start) throw new Error("Repository has no commit to check out"); const cwd = await canonicalPath(input.cwd); - await removeStaleRegistrations(input.base, cwd); + await removeStaleNestedRegistrations(input.base, cwd); await input.beforeAdd?.().catch((error: unknown) => { throw new NestedWorktreeAuthorizationFailure(error); }); if (!branch) { await git(input.base, ["worktree", "add", "--quiet", "--detach", cwd, start], { timeout: NESTED_WORKTREE_GIT_TIMEOUT_MS }); @@ -241,7 +241,7 @@ export async function restoreNestedRepositoryWorktree(input: { const cwd = await canonicalPath(input.cwd); const prefix = await nestedWorktreeBranchPrefix(cwd); if (input.branchName && !input.branchName.startsWith(prefix)) throw new Error("unowned_nested_worktree_branch"); - await removeStaleRegistrations(input.base, cwd); + await removeStaleNestedRegistrations(input.base, cwd); const existing = input.branchName ? (await git(input.base, ["rev-parse", "--verify", "-q", `refs/heads/${input.branchName}`]).catch(() => "")).trim() : ""; if (existing && existing !== input.commit) throw new Error("nested_worktree_branch_moved"); @@ -262,7 +262,7 @@ export async function restoreNestedRepositoryWorktree(input: { await input.afterAdd(cwd); } catch (error) { await git(input.base, ["worktree", "remove", "--force", cwd]).catch(() => undefined); - await removeStaleRegistrations(input.base, cwd).catch(() => undefined); + await removeStaleNestedRegistrations(input.base, cwd).catch(() => undefined); if (input.branchName && !existing) { await git(input.base, ["update-ref", "-d", `refs/heads/${input.branchName}`, input.commit]).catch(() => undefined); } @@ -310,7 +310,7 @@ export async function releaseNestedRepositoryWorktrees(worktrees: LinkedNestedCh try { await withNestedRepositoryBaseLock(worktree.base, async () => { if (!await missing(worktree.checkout)) return; - await removeStaleRegistrations(worktree.base, worktree.checkout); + await removeStaleNestedRegistrations(worktree.base, worktree.checkout); const refs = (await git(worktree.base, ["for-each-ref", "--format=%(objectname) %(refname)", `refs/heads/${worktree.branchPrefix}`])) .split("\n").map(line => line.trim().split(" ")).filter(([sha, ref]) => /^[a-f0-9]{40}$/.test(sha ?? "") && ref); const held = await checkedOutBranches(worktree.base); @@ -331,3 +331,15 @@ export async function releaseNestedRepositoryWorktrees(worktrees: LinkedNestedCh export async function repairMovedNestedRepositoryWorktree(base: string, moved: string) { await withNestedRepositoryBaseLock(base, () => git(base, ["worktree", "repair", moved])); } + +export async function moveNestedRepositoryWorktree(base: string, from: string, to: string) { + await withNestedRepositoryBaseLock(base, async () => { + await git(base, ["worktree", "move", from, to], { timeout: NESTED_WORKTREE_GIT_TIMEOUT_MS }); + await git(base, ["worktree", "repair", to]); + }); +} + +export async function trackNestedWorktreeUpstream(base: string, branchName: string, upstream: string | null) { + if (!upstream || !(await git(base, ["rev-parse", "--verify", "-q", `refs/remotes/origin/${upstream}^{commit}`]).catch(() => "")).trim()) return; + await git(base, ["branch", "--quiet", `--set-upstream-to=origin/${upstream}`, branchName]); +}