diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index bcd8156666e..5e208871f66 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -2592,8 +2592,20 @@ Before deletion the worker fetches the exact commit into a fresh object store using the company's normal Git identity and commits a versioned restoration recipe to workspace metadata. Current runs, previews, pending reopening, unresolved recovery, dirty/untracked files, unknown ignored files, unproven -ownership, and unsupported or unique Git state prevent deletion. Only the -existing repository-ignored regenerable-output allowlist can be discarded. +ownership, and unsupported or unique Git state prevent deletion. Ignored files +can be discarded only when they are repository-ignored regenerable output, +Paperclip's `.claude/settings.local.json` (rewritten on every Claude run), or a +managed project checkout under `.paperclip-repositories/-` that the +next run re-clones. A checkout qualifies only when its name maps back to one of +the project's repositories and it passes the same checks as the worktree +(clean tree, no hidden index flags, allowed local config, no in-progress +state), stock Git storage (no custom hooks), no linked worktrees, no +unreachable objects, and every commit reachable from its refs or reflogs is +reachable from a current upstream branch or tag, read fresh from the project +repository's URL with the company's Git identity. Task repositories +(`task-repo-*`) always prevent deletion. Inspection runs Git with +`core.fsmonitor` disabled, so a repository's local config cannot make it run a +command. Deletion first renames the worktree to recorded trash; restart and reopening finish that cleanup under the same source/lifecycle locks. diff --git a/server/src/__tests__/git-reproducible-managed-checkouts.test.ts b/server/src/__tests__/git-reproducible-managed-checkouts.test.ts new file mode 100644 index 00000000000..4cba912d779 --- /dev/null +++ b/server/src/__tests__/git-reproducible-managed-checkouts.test.ts @@ -0,0 +1,212 @@ +import { execFile as execFileCallback } from "node:child_process"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + assertReproducibleGitFiles, + assertReproducibleManagedCheckout, + type ManagedCheckoutProver, +} from "../services/git-workspace-restoration.js"; + +const execFile = promisify(execFileCallback); + +async function git(cwd: string, ...args: string[]) { + const { stdout } = await execFile("git", ["-C", cwd, "-c", "user.name=Test", "-c", "user.email=test@example.com", + "-c", "init.defaultBranch=main", "-c", "commit.gpgsign=false", ...args]); + return stdout; +} + +async function write(file: string, content = "x\n") { + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, content); +} + +async function commit(cwd: string, file: string, message: string) { + await write(path.join(cwd, file), `${message}\n`); + await git(cwd, "add", file); + await git(cwd, "commit", "-q", "-m", message); + // Committing leaves COMMIT_EDITMSG, which the storage check rejects on its + // own; remove it so these tests reach the history checks. + await fs.rm(path.join(cwd, ".git", "COMMIT_EDITMSG"), { force: true }); + return (await git(cwd, "rev-parse", "HEAD")).trim(); +} + +describe("assertReproducibleGitFiles with Paperclip-managed files", () => { + let root: string; + let work: string; + let projectSeed: string; + let projectRemote: string; + let prove: ManagedCheckoutProver; + + // A bare upstream with one commit, plus the seed it was pushed from. + async function upstream(name: string, files: Record) { + const seed = path.join(root, `${name}-seed`); + await fs.mkdir(seed, { recursive: true }); + await git(seed, "init", "-q"); + for (const [file, content] of Object.entries(files)) await write(path.join(seed, file), content); + await git(seed, "add", "-A"); + await git(seed, "commit", "-q", "-m", "seed"); + const bare = path.join(root, `${name}.git`); + await execFile("git", ["clone", "-q", "--bare", seed, bare]); + await git(seed, "remote", "add", "origin", bare); + return { seed, bare }; + } + + beforeEach(async () => { + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-reproducible-"))); + const main = await upstream("main", { + "README.md": "main\n", + ".gitignore": "node_modules/\n.claude/settings.local.json\nsecret.txt\n", + }); + ({ seed: projectSeed, bare: projectRemote } = await upstream("project", { "README.md": "project\n", ".gitignore": "node_modules/\n" })); + work = path.join(root, "work"); + await execFile("git", ["clone", "-q", main.bare, work]); + // Workspace preparation excludes the managed namespace through info/exclude. + await fs.appendFile(path.join(work, ".git", "info", "exclude"), "\n/.paperclip-repositories/\n"); + prove = (checkout, _name, signal) => assertReproducibleManagedCheckout(checkout, { + repoUrl: projectRemote, resolveGitAuth: async () => null, signal, + }); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + async function managedCheckout(name = "project-d051a9c4e671") { + const checkout = path.join(work, ".paperclip-repositories", name); + await fs.mkdir(path.dirname(checkout), { recursive: true }); + await execFile("git", ["clone", "-q", projectRemote, checkout]); + return checkout; + } + + const check = () => assertReproducibleGitFiles(work, undefined, { proveManagedCheckout: prove }); + const expectAccepted = () => expect(check()).resolves.toBeDefined(); + const expectRejected = (reason: string | RegExp) => expect(check()).rejects.toThrow(reason); + + it("accepts Paperclip's Claude settings file", async () => { + await write(path.join(work, ".claude", "settings.local.json"), "{}\n"); + await expectAccepted(); + }); + + it("accepts a clean managed checkout, including its regenerable directories", async () => { + const checkout = await managedCheckout(); + await write(path.join(checkout, "node_modules", "pkg", "index.js")); + await write(path.join(work, "node_modules", "pkg", "index.js")); + await expectAccepted(); + }); + + it("rejects managed checkouts unless the caller can prove them", async () => { + await managedCheckout(); + await expect(assertReproducibleGitFiles(work)).rejects.toThrow("unique_ignored_files"); + }); + + it("still rejects unknown ignored files", async () => { + await write(path.join(work, "secret.txt")); + await expectRejected("unique_ignored_files"); + }); + + it("rejects changed, untracked, and flag-hidden edits", async () => { + const checkout = await managedCheckout(); + await write(path.join(checkout, "README.md"), "edited\n"); + await expectRejected("dirty_or_untracked_files"); + await git(checkout, "checkout", "-q", "--", "README.md"); + await write(path.join(checkout, "notes.txt")); + await expectRejected("dirty_or_untracked_files"); + await fs.rm(path.join(checkout, "notes.txt")); + // skip-worktree hides this edit from `git status`. + await git(checkout, "update-index", "--skip-worktree", "README.md"); + await write(path.join(checkout, "README.md"), "hidden edit\n"); + await expectRejected("unsupported_index_flags"); + }); + + it("rejects commits kept by a branch, a tag, or only the reflog", async () => { + const checkout = await managedCheckout(); + await commit(checkout, "branch.txt", "branch work"); + await expectRejected("unpushed_managed_checkout"); + await git(checkout, "tag", "kept"); + // Loose tag refs fail the storage check on their own; packed ones reach the history check. + await git(checkout, "pack-refs", "--all"); + await git(checkout, "reset", "-q", "--hard", "origin/main"); + // ORIG_HEAD alone would already reject; drop it to reach the history check. + await fs.rm(path.join(checkout, ".git", "ORIG_HEAD"), { force: true }); + await git(checkout, "reflog", "expire", "--expire=now", "--all"); + await expectRejected("unpushed_managed_checkout"); + await git(checkout, "tag", "-d", "kept"); + await commit(checkout, "amended.txt", "amended work"); + await git(checkout, "reset", "-q", "--hard", "origin/main"); + // ORIG_HEAD alone would already reject; drop it to reach the history checks. + await fs.rm(path.join(checkout, ".git", "ORIG_HEAD"), { force: true }); + await expectRejected(/unique_reflog_commits|unpushed_managed_checkout/); + }); + + it("rejects a detached HEAD commit after its reflog expires", async () => { + const checkout = await managedCheckout(); + await git(checkout, "checkout", "-q", "--detach"); + await commit(checkout, "detached.txt", "detached work"); + await git(checkout, "reflog", "expire", "--expire=now", "--all"); + await expectRejected("unpushed_managed_checkout"); + }); + + it("accepts pushed history after upstream advances", async () => { + await managedCheckout(); + // Upstream moves on after the clone; the checkout never fetches it. + await commit(projectSeed, "later.txt", "later upstream work"); + await git(projectSeed, "push", "-q", "origin", "main"); + await expectAccepted(); + }); + + it("does not trust a stale remote-tracking ref", async () => { + await git(projectSeed, "checkout", "-q", "-b", "feature"); + await commit(projectSeed, "feature.txt", "feature work"); + await git(projectSeed, "push", "-q", "origin", "feature"); + await managedCheckout(); + // The branch disappears upstream; the checkout's origin/feature is now the last copy. + await git(projectRemote, "branch", "-q", "-D", "feature"); + await expectRejected("unpushed_managed_checkout"); + }); + + it("rejects custom hooks, linked worktrees, stashes, and unreachable objects", async () => { + const checkout = await managedCheckout(); + await write(path.join(checkout, ".git", "hooks", "pre-commit"), "#!/bin/sh\nexit 0\n"); + await expectRejected("custom_git_metadata"); + await fs.rm(path.join(checkout, ".git", "hooks", "pre-commit")); + + const linked = path.join(root, "linked"); + await git(checkout, "worktree", "add", "-q", "--detach", linked); + await expectRejected("managed_checkout_has_worktrees"); + await git(checkout, "worktree", "remove", "--force", linked); + + await write(path.join(checkout, "README.md"), "stashed\n"); + await git(checkout, "stash", "-q"); + await expectRejected(/unsupported_local_git_state|unsupported_managed_checkout_refs/); + await git(checkout, "stash", "clear"); + + await write(path.join(root, "orphan.txt"), "only copy\n"); + await git(checkout, "hash-object", "-w", path.join(root, "orphan.txt")); + await expectRejected("unreachable_managed_checkout_objects"); + }); + + it("never runs a configured fsmonitor command while inspecting", async () => { + const checkout = await managedCheckout(); + const marker = path.join(root, "fsmonitor-ran"); + const hook = path.join(root, "fsmonitor.sh"); + await write(hook, `#!/bin/sh\ntouch '${marker}'\nexit 1\n`); + await fs.chmod(hook, 0o755); + for (const repository of [checkout, work]) { + await git(repository, "config", "core.fsmonitor", hook); + await check().catch(() => undefined); + await git(repository, "config", "--unset", "core.fsmonitor"); + } + await expect(fs.access(marker)).rejects.toThrow(); + }); + + it("never treats task repositories or plain directories as reproducible", async () => { + await managedCheckout("task-repo-0123456789abcdef01234567"); + await expectRejected("unique_ignored_files"); + await fs.rm(path.join(work, ".paperclip-repositories"), { recursive: true }); + await write(path.join(work, ".paperclip-repositories", "project-d051a9c4e671", "README.md")); + await expectRejected("unproven_managed_checkout"); + }); +}); diff --git a/server/src/services/git-workspace-expiration.ts b/server/src/services/git-workspace-expiration.ts index 42f8e12c209..4a5c56edbef 100644 --- a/server/src/services/git-workspace-expiration.ts +++ b/server/src/services/git-workspace-expiration.ts @@ -5,8 +5,10 @@ import { executionWorkspaces, issues, heartbeatRuns, nativeRunFinalizations, env workspaceRuntimeServices, issueRecoveryActions, projectWorkspaces, withDedicatedDbConnection, type Db } from "@paperclipai/db"; import { createGitRemoteAuthProvider } from "./git-credentials.js"; import { withIsolatedRepositorySourceLock, verifiedRecordedIsolatedRemoteSource } from "./isolated-remote-workspace-source.js"; -import { assertReproducibleGitFiles, assertReproducibleBaseGitStorage, inspectGitCacheRecipe, readGitCacheRestoration, - verifyRemoteGitCommit, gitCacheCommand, readGitCacheExpiration, finishGitCacheTrash, type GitCacheExpiration, GIT_CACHE_RESTORATION_KEY, GIT_CACHE_EXPIRATION_REASON } from "./git-workspace-restoration.js"; +import { managedProjectCheckoutName } from "./managed-project-checkout-name.js"; +import type { GitRemoteAuthProvider } from "./git-credentials.js"; +import { assertReproducibleGitFiles, assertReproducibleBaseGitStorage, assertReproducibleManagedCheckout, inspectGitCacheRecipe, readGitCacheRestoration, + verifyRemoteGitCommit, type ManagedCheckoutProver, gitCacheCommand, readGitCacheExpiration, finishGitCacheTrash, type GitCacheExpiration, GIT_CACHE_RESTORATION_KEY, GIT_CACHE_EXPIRATION_REASON } from "./git-workspace-restoration.js"; async function allocatedBytes(target: string, shouldContinue: () => boolean): Promise { @@ -122,6 +124,26 @@ async function inspectBase(db: Db, workspace: Workspace, signal?: AbortSignal, a return allocatedBytes(recipe.sourcePath, () => { signal?.throwIfAborted(); return true; }); } +/** + * Accepts a managed project checkout only when its name maps back to one of + * the workspace's project repositories (workspace preparation re-clones it + * from that row on the next run) and the checkout is proven reproducible + * against that row's repository URL. + */ +function managedCheckoutProver(db: Db, workspace: Workspace, auth: GitRemoteAuthProvider): ManagedCheckoutProver { + let rows: Promise> | undefined; + return async (checkoutPath, name, signal) => { + if (!workspace.projectId) throw new Error("unknown_managed_checkout"); + rows ??= db.select({ id: projectWorkspaces.id, repoUrl: projectWorkspaces.repoUrl, repoRef: projectWorkspaces.repoRef, + cwd: projectWorkspaces.cwd }).from(projectWorkspaces) + .where(and(eq(projectWorkspaces.companyId, workspace.companyId), eq(projectWorkspaces.projectId, workspace.projectId))); + const source = (await rows).find(row => row.repoUrl + && managedProjectCheckoutName({ id: row.id, repoUrl: row.repoUrl, repoRef: row.repoRef, cwd: row.cwd }) === name); + if (!source?.repoUrl) throw new Error("unknown_managed_checkout"); + await assertReproducibleManagedCheckout(checkoutPath, { repoUrl: source.repoUrl, resolveGitAuth: auth, signal }); + }; +} + export async function expireReconstructableGitWorkspace(db: Db, snapshot: Workspace, options: { dryRun: boolean; expireBase: boolean; now: Date; signal?: AbortSignal; beforeDelete?: (workspace: Workspace) => Promise; @@ -141,8 +163,10 @@ export async function expireReconstructableGitWorkspace(db: Db, snapshot: Worksp // A restored active workspace starts a new expiration cycle and records its new HEAD. if (!hasPendingGitExpiration(workspace) && !["active", "idle", "in_review"].includes(workspace.status)) throw new Error("not_expirable"); if (["active", "idle", "in_review"].includes(workspace.status)) state = null; + const auth = createGitRemoteAuthProvider(db, workspace.companyId, { issueId: workspace.sourceIssueId! }); + const proveManagedCheckout = managedCheckoutProver(db, workspace, auth); let recipe = state ? readGitCacheRestoration(workspace)! : await inspectGitCacheRecipe(workspace, - createGitRemoteAuthProvider(db, workspace.companyId, { issueId: workspace.sourceIssueId! }), options.signal); + auth, options.signal, { proveManagedCheckout }); const present = await exists(recipe.worktreePath); if (!state && !present) throw new Error("missing_unverified_worktree"); result.candidate = true; @@ -185,7 +209,7 @@ export async function expireReconstructableGitWorkspace(db: Db, snapshot: Worksp const { acquireGitWorktreeCleanupLock } = await import("./workspace-runtime.js"); const lock = await acquireGitWorktreeCleanupLock(recipe.worktreePath); try { - await assertReproducibleGitFiles(recipe.worktreePath, options.signal); + await assertReproducibleGitFiles(recipe.worktreePath, options.signal, { proveManagedCheckout }); if ((await gitCacheCommand(recipe.worktreePath, ["rev-parse", "HEAD"], options.signal)).trim() !== recipe.commit) throw new Error("head_changed"); await fs.rename(recipe.worktreePath, state.worktreeTrash); await finishGitCacheTrash(workspace); diff --git a/server/src/services/git-workspace-restoration.ts b/server/src/services/git-workspace-restoration.ts index 19fd71eeecd..2675bb7870b 100644 --- a/server/src/services/git-workspace-restoration.ts +++ b/server/src/services/git-workspace-restoration.ts @@ -10,6 +10,7 @@ import type { GitRemoteAuthProvider } from "./git-credentials.js"; import { isRuntimeOwnedGitBranch } from "./execution-workspace-branch-ownership.js"; import { recordedIsolatedRemoteSource, verifiedRecordedIsolatedRemoteSource, materializeIsolatedRemoteWorkspaceSource } from "./isolated-remote-workspace-source.js"; import { findRegenerableWorkspaceDirs } from "./workspace-compaction.js"; +import { PROJECT_REPOSITORIES_DIR } from "@paperclipai/adapter-utils/git-workspace-sync"; const execFile = promisify(execFileCallback); const shaPattern = /^[a-f0-9]{40}$/; @@ -92,7 +93,10 @@ export function readGitCacheRestoration(workspace: Workspace): GitCacheRestorati } export async function gitCacheCommand(cwd: string, args: string[], signal?: AbortSignal) { - const result = await execFile("git", ["-C", cwd, ...args], { + // The inspected repositories are agent-writable. core.fsmonitor in their + // local config would make `git status` run an arbitrary command as the + // server; the command-line setting takes precedence over it. + const result = await execFile("git", ["-c", "core.fsmonitor=false", "-C", cwd, ...args], { timeout: 60_000, signal, maxBuffer: 16 * 1024 * 1024, env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_OPTIONAL_LOCKS: "0" }, }); @@ -170,15 +174,128 @@ export async function assertReproducibleBaseGitStorage(cwd: string, signal?: Abo for (const directory of ["objects", "refs", "logs"]) await walk(directory); } -/** Inspect every untracked/ignored file. Only repository-ignored regenerable directories qualify. */ -export async function assertReproducibleGitFiles(cwd: string, signal?: AbortSignal) { +/** + * Paperclip writes this file into the worktree on every Claude run (it merges + * its permission rules into whatever is there), so a terminal worktree's copy + * is regenerated when the task runs again. + */ +const PAPERCLIP_CLAUDE_SETTINGS_FILE = ".claude/settings.local.json"; + +/** + * Proves that a managed project checkout (`.paperclip-repositories/`) + * holds nothing that re-cloning would not restore; throws otherwise. Only the + * expiration worker supplies one, because it can map the checkout back to its + * project repository and reach upstream. + */ +export type ManagedCheckoutProver = (checkoutPath: string, name: string, signal?: AbortSignal) => Promise; + +/** + * The proof for one managed checkout, against the repository URL recorded on + * its project workspace row (never the checkout's own, agent-writable config): + * the checkout's tree, index, config and Git storage hold nothing unique, it + * has no linked worktrees, and every commit reachable from its refs or + * reflogs is reachable from a current upstream branch or tag. + */ +export async function assertReproducibleManagedCheckout(checkout: string, input: { + repoUrl: string; resolveGitAuth: GitRemoteAuthProvider; signal?: AbortSignal; +}) { + const { signal } = input; + const gitDir = await fs.lstat(path.join(checkout, ".git")).catch(() => null); + if (!gitDir?.isDirectory()) throw new Error("unproven_managed_checkout"); + if ((await gitCacheCommand(checkout, ["rev-parse", "--show-toplevel"], signal)).trim() !== await fs.realpath(checkout)) { + throw new Error("unproven_managed_checkout"); + } + const worktrees = (await gitCacheCommand(checkout, ["worktree", "list", "--porcelain"], signal)) + .split("\n").filter(line => line.startsWith("worktree ")); + if (worktrees.length !== 1) throw new Error("managed_checkout_has_worktrees"); + // Tree, index flags, local config, ignored files and in-progress state, as + // for any worktree; nested managed checkouts are never accepted here. + await assertReproducibleGitFiles(checkout, signal); + // Hooks, info files and object/ref storage must be stock. + await assertReproducibleBaseGitStorage(checkout, signal); + const refs = (await gitCacheCommand(checkout, ["for-each-ref", "--format=%(refname)"], signal)).trim().split("\n").filter(Boolean); + if (refs.some(ref => !/^refs\/(heads|tags|remotes\/origin)\//.test(ref))) throw new Error("unsupported_managed_checkout_refs"); + // Every commit a ref or reflog keeps must be on a current upstream branch or + // tag. Local remote-tracking refs prove nothing on their own (they may be + // stale), so the proof fetches upstream's refs fresh. + // HEAD is listed explicitly: a detached HEAD is not a named ref, and once + // its reflog expires nothing else would name its commits. + const localTips = new Set([ + ...(await gitCacheCommand(checkout, ["rev-parse", "--verify", "HEAD^{commit}"], signal)).split("\n"), + ...(await gitCacheCommand(checkout, ["for-each-ref", "--format=%(objectname)"], signal)).split("\n"), + ...(await gitCacheCommand(checkout, ["reflog", "show", "--all", "--format=%H"], signal).catch(() => "")).split("\n"), + ].map(line => line.trim()).filter(sha => shaPattern.test(sha))); + if (localTips.size > MANAGED_CHECKOUT_LOCAL_TIP_LIMIT) throw new Error("managed_checkout_local_tip_budget"); + if (localTips.size && await hasCommitsMissingUpstream(checkout, [...localTips], input.repoUrl, input.resolveGitAuth, signal)) { + throw new Error("unpushed_managed_checkout"); + } + // Objects no ref or reflog reaches (a reset `git add`, a dropped stash) may + // be the only copy of some work. + const unreachable = await gitCacheCommand(checkout, ["fsck", "--connectivity-only", "--unreachable", "--no-progress"], signal); + if (unreachable.trim()) throw new Error("unreachable_managed_checkout_objects"); +} + +/** Upper bound on distinct ref and reflog commits a proof will consider. */ +const MANAGED_CHECKOUT_LOCAL_TIP_LIMIT = 5_000; + +/** + * Fetches upstream's current branches and tags into a throwaway proof store + * that borrows the checkout's objects (so only commits the checkout lacks are + * downloaded), then asks whether any local commit is unreachable from them. + * It is one fetch and one rev-list however many refs either side has. The + * fetch runs config-free with the company's Git identity, never the + * checkout's own config. + */ +async function hasCommitsMissingUpstream(checkout: string, localTips: string[], repoUrl: string, + resolveGitAuth: GitRemoteAuthProvider, signal?: AbortSignal) { + const store = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-git-checkout-proof-")); + try { + const auth = await resolveGitAuth(repoUrl); + const env = { ...process.env, ...auth?.env, GIT_TERMINAL_PROMPT: "0", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: os.devNull }; + const options = { timeout: 120_000, signal, maxBuffer: 16 * 1024 * 1024, env, cwd: os.tmpdir() }; + await execFile("git", ["init", "--bare", "--quiet", store], options); + await fs.writeFile(path.join(store, "objects", "info", "alternates"), `${path.join(await fs.realpath(checkout), ".git", "objects")}\n`); + await execFile("git", [...auth?.configArgs ?? [], "-C", store, "fetch", "--quiet", "--no-tags", "--no-recurse-submodules", + "--", repoUrl, "+refs/heads/*:refs/upstream/heads/*", "+refs/tags/*:refs/upstream/tags/*"], options); + const missing = await execFile("git", ["-C", store, "rev-list", "--max-count=1", ...localTips, "--not", "--glob=refs/upstream/*"], options); + return missing.stdout.trim() !== ""; + } finally { + await fs.rm(store, { recursive: true, force: true }); + } +} + +/** + * Inspect every untracked/ignored file. Only repository-ignored regenerable + * directories, Paperclip's Claude settings file, and managed project checkouts + * that `proveManagedCheckout` accepts qualify. + */ +export async function assertReproducibleGitFiles(cwd: string, signal?: AbortSignal, options: { + proveManagedCheckout?: ManagedCheckoutProver; +} = {}) { if ((await gitCacheCommand(cwd, ["status", "--porcelain=v1", "--untracked-files=all", "-z"], signal)).length) throw new Error("dirty_or_untracked_files"); if ((await gitCacheCommand(cwd, ["ls-files", "-v"], signal)).split("\n").some(line => /^[a-zS] /.test(line))) throw new Error("unsupported_index_flags"); const localConfig = await gitCacheCommand(cwd, ["config", "--local", "--name-only", "--list"], signal); if (localConfig.split("\n").filter(Boolean).some(key => !/^(core\.(repositoryformatversion|filemode|bare|logallrefupdates|ignorecase|precomposeunicode)|remote\.origin\.(url|fetch)|branch\.[^.]+\.(remote|merge))$/.test(key))) throw new Error("unsupported_local_git_config"); const regenerable = await findRegenerableWorkspaceDirs(cwd, { signal }); const ignored = (await gitCacheCommand(cwd, ["ls-files", "--others", "--ignored", "--exclude-standard", "-z"], signal)).split("\0").filter(Boolean); - if (ignored.some(file => !regenerable.some(dir => file.startsWith(`${dir}/`)))) throw new Error("unique_ignored_files"); + const unexplained = ignored.filter(file => file !== PAPERCLIP_CLAUDE_SETTINGS_FILE && !regenerable.some(dir => file.startsWith(`${dir}/`))); + // Anything else unexplained must sit inside a managed project checkout that + // the caller can prove reproducible. Task repositories (`task-repo-*`) carry + // access receipts and staging clones (`*.clone-*`) are mid-publication, so + // neither ever qualifies. + const checkouts = new Set(); + for (const file of unexplained) { + const [namespace, name, rest] = file.split("/", 3); + if (namespace !== PROJECT_REPOSITORIES_DIR || !name || rest === undefined || !options.proveManagedCheckout + || name.startsWith("task-repo-") || name.includes(".clone-")) throw new Error("unique_ignored_files"); + checkouts.add(name); + } + for (const name of checkouts) { + const checkout = path.join(cwd, PROJECT_REPOSITORIES_DIR, name); + const entry = await fs.lstat(checkout); + if (!entry.isDirectory() || entry.isSymbolicLink()) throw new Error("unique_ignored_files"); + await options.proveManagedCheckout!(checkout, name, signal); + } const entries = await gitCacheCommand(cwd, ["ls-tree", "-r", "HEAD"], signal); if (/^160000 /m.test(entries)) throw new Error("unsupported_submodules"); const gitDir = (await gitCacheCommand(cwd, ["rev-parse", "--absolute-git-dir"], signal)).trim(); @@ -222,7 +339,8 @@ export async function assertReproducibleGitFiles(cwd: string, signal?: AbortSign return regenerable; } -export async function inspectGitCacheRecipe(workspace: Workspace, resolveGitAuth: GitRemoteAuthProvider, signal?: AbortSignal): Promise { +export async function inspectGitCacheRecipe(workspace: Workspace, resolveGitAuth: GitRemoteAuthProvider, signal?: AbortSignal, + options: { proveManagedCheckout?: ManagedCheckoutProver } = {}): Promise { if (workspace.mode !== "isolated_workspace" || workspace.strategyType !== "git_worktree" || !isRuntimeOwnedGitBranch(workspace.metadata) || !workspace.sourceIssueId || !workspace.repoUrl || !workspace.cwd || !workspace.branchName || (workspace.providerRef && workspace.providerRef !== workspace.cwd)) throw new Error("unproven_workspace_ownership"); @@ -241,7 +359,7 @@ export async function inspectGitCacheRecipe(workspace: Workspace, resolveGitAuth if (!(await fs.lstat(path.join(workspace.cwd, ".git"))).isFile()) throw new Error("not_linked_worktree"); if ((await gitCacheCommand(workspace.cwd, ["rev-parse", "--path-format=absolute", "--git-common-dir"], signal)).trim() !== path.join(await fs.realpath(source), ".git")) throw new Error("source_identity_changed"); if ((await gitCacheCommand(workspace.cwd, ["symbolic-ref", "--short", "HEAD"], signal)).trim() !== workspace.branchName) throw new Error("branch_identity_changed"); - await assertReproducibleGitFiles(workspace.cwd, signal); + await assertReproducibleGitFiles(workspace.cwd, signal, options); const commit = (await gitCacheCommand(workspace.cwd, ["rev-parse", "HEAD"], signal)).trim(); await verifyRemoteGitCommit(workspace.repoUrl, commit, resolveGitAuth, signal); return { schema: "paperclip.git-cache-restoration.v1", companyId: workspace.companyId, workspaceId: workspace.id, diff --git a/server/src/services/heartbeat/workspaces.ts b/server/src/services/heartbeat/workspaces.ts index fea3cfb32d1..f76b54ffaec 100644 --- a/server/src/services/heartbeat/workspaces.ts +++ b/server/src/services/heartbeat/workspaces.ts @@ -1,4 +1,5 @@ import { assertIsolatedWorkspaceSourceReadAccess, retainIsolatedWorkspaceSourceAccess } from "../execution-workspace-repositories.js"; +import { deriveRepoNameFromRepoUrl, managedProjectCheckoutName, projectRepositoryIdentity } from "../managed-project-checkout-name.js"; import { forbidden } from "../../errors.js"; import { readGitConnectionFailure } from "../git-connection-failure.js"; import { materializeManagedProjectWorkspace, ensureManagedRepositoriesIgnored } from "../managed-repository-checkout.js"; @@ -423,24 +424,6 @@ export async function resolveWorkspaceAfterLowTrustPreflight< }; } -function deriveRepoNameFromRepoUrl(repoUrl: string | null): string | null { - const trimmed = repoUrl?.trim() ?? ""; - if (!trimmed) return null; - try { - const parsed = new URL(trimmed); - const cleanedPath = parsed.pathname.replace(/\/+$/, ""); - const repoName = - cleanedPath - .split("/") - .filter(Boolean) - .pop() - ?.replace(/\.git$/i, "") ?? ""; - return repoName || null; - } catch { - return null; - } -} - type PrepareManagedLocalSource = (sourceCwd: string, targetCwd: string) => Promise<(targetCwd: string) => Promise>; /** @@ -516,7 +499,7 @@ export async function prepareProjectRepositoryWorkspaces(input: { beforePublish?: () => Promise; prepareLocalSource?: (sourceCwd: string) => Promise<() => Promise>; }): Promise> { - const identity = (url: string) => url.trim().replace(/\.git\/?$/, "").replace(/\/$/, ""); + const identity = projectRepositoryIdentity; const seen = new Set(input.anchorRepoUrl ? [identity(input.anchorRepoUrl)] : []); const selected = input.workspaces.filter((workspace) => { if (!workspace.repoUrl || seen.has(identity(workspace.repoUrl))) return false; @@ -533,8 +516,7 @@ export async function prepareProjectRepositoryWorkspaces(input: { const results = []; for (const workspace of selected) { const repoUrl = workspace.repoUrl!; - const name = (deriveRepoNameFromRepoUrl(repoUrl) ?? "repo").replace(/[^a-zA-Z0-9_-]/g, "-"); - const key = `${name}-${createHash("sha256").update(JSON.stringify([workspace.id, identity(repoUrl), workspace.repoRef, workspace.cwd ?? null])).digest("hex").slice(0, 12)}`; + const key = managedProjectCheckoutName({ id: workspace.id, repoUrl, repoRef: workspace.repoRef, cwd: workspace.cwd }); const cwd = path.join(root, key); const existing = await fs.lstat(cwd).catch(() => null); if (existing && (!existing.isDirectory() || existing.isSymbolicLink())) throw new Error("Invalid project repository checkout path"); diff --git a/server/src/services/managed-project-checkout-name.ts b/server/src/services/managed-project-checkout-name.ts new file mode 100644 index 00000000000..8ede84259ea --- /dev/null +++ b/server/src/services/managed-project-checkout-name.ts @@ -0,0 +1,40 @@ +import { createHash } from "node:crypto"; + +/** Normalizes a repository URL so `.git` and trailing-slash variants match. */ +export function projectRepositoryIdentity(url: string) { + return url.trim().replace(/\.git\/?$/, "").replace(/\/$/, ""); +} + +export function deriveRepoNameFromRepoUrl(repoUrl: string | null): string | null { + const trimmed = repoUrl?.trim() ?? ""; + if (!trimmed) return null; + try { + const parsed = new URL(trimmed); + const cleanedPath = parsed.pathname.replace(/\/+$/, ""); + const repoName = + cleanedPath + .split("/") + .filter(Boolean) + .pop() + ?.replace(/\.git$/i, "") ?? ""; + return repoName || null; + } catch { + return null; + } +} + +/** + * Directory name of a project repository's managed checkout under + * `.paperclip-repositories/`. It is derived from the project workspace row, + * so the same row always maps to the same checkout, and a checkout name maps + * back to the row it was cloned from. + */ +export function managedProjectCheckoutName(workspace: { + id: string; repoUrl: string; repoRef: string | null; cwd?: string | null; +}) { + const name = (deriveRepoNameFromRepoUrl(workspace.repoUrl) ?? "repo").replace(/[^a-zA-Z0-9_-]/g, "-"); + const hash = createHash("sha256") + .update(JSON.stringify([workspace.id, projectRepositoryIdentity(workspace.repoUrl), workspace.repoRef, workspace.cwd ?? null])) + .digest("hex").slice(0, 12); + return `${name}-${hash}`; +}