mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-21 08:22:20 +02:00
fix(server): let Git expiration discard Paperclip-managed checkouts (#16586)
Reconstructable Git expiration rejected nearly every terminal worktree with unique_ignored_files, because Paperclip writes two ignored paths into every worktree itself: .claude/settings.local.json and the managed project checkouts under .paperclip-repositories/. Expiration now accepts the settings file, and accepts a managed checkout only through a prover the expiration worker supplies. The prover maps the checkout back to its project workspace row and proves it reproducible against that row's repository URL: worktree and stock-storage checks, no linked worktrees, no unreachable objects, and every ref, reflog and HEAD commit reachable from upstream refs fetched fresh into a proof store. Every inspection Git command now runs with core.fsmonitor disabled. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
3db33c131e
commit
73659c6b27
6 files changed
+421
-33
No files matched your search
+14
-2
@@ -2592,8 +2592,20 @@ Before deletion the worker fetches the exact commit into a fresh object store
|
||||
using the company's normal Git identity and commits a versioned restoration
|
||||
recipe to workspace metadata. Current runs, previews, pending reopening,
|
||||
unresolved recovery, dirty/untracked files, unknown ignored files, unproven
|
||||
ownership, and unsupported or unique Git state prevent deletion. Only the
|
||||
existing repository-ignored regenerable-output allowlist can be discarded.
|
||||
ownership, and unsupported or unique Git state prevent deletion. Ignored files
|
||||
can be discarded only when they are repository-ignored regenerable output,
|
||||
Paperclip's `.claude/settings.local.json` (rewritten on every Claude run), or a
|
||||
managed project checkout under `.paperclip-repositories/<repo>-<hash>` that the
|
||||
next run re-clones. A checkout qualifies only when its name maps back to one of
|
||||
the project's repositories and it passes the same checks as the worktree
|
||||
(clean tree, no hidden index flags, allowed local config, no in-progress
|
||||
state), stock Git storage (no custom hooks), no linked worktrees, no
|
||||
unreachable objects, and every commit reachable from its refs or reflogs is
|
||||
reachable from a current upstream branch or tag, read fresh from the project
|
||||
repository's URL with the company's Git identity. Task repositories
|
||||
(`task-repo-*`) always prevent deletion. Inspection runs Git with
|
||||
`core.fsmonitor` disabled, so a repository's local config cannot make it run a
|
||||
command.
|
||||
Deletion first renames the worktree to recorded trash; restart and reopening
|
||||
finish that cleanup under the same source/lifecycle locks.
|
||||
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
import { execFile as execFileCallback } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
assertReproducibleGitFiles,
|
||||
assertReproducibleManagedCheckout,
|
||||
type ManagedCheckoutProver,
|
||||
} from "../services/git-workspace-restoration.js";
|
||||
|
||||
const execFile = promisify(execFileCallback);
|
||||
|
||||
async function git(cwd: string, ...args: string[]) {
|
||||
const { stdout } = await execFile("git", ["-C", cwd, "-c", "user.name=Test", "-c", "user.email=test@example.com",
|
||||
"-c", "init.defaultBranch=main", "-c", "commit.gpgsign=false", ...args]);
|
||||
return stdout;
|
||||
}
|
||||
|
||||
async function write(file: string, content = "x\n") {
|
||||
await fs.mkdir(path.dirname(file), { recursive: true });
|
||||
await fs.writeFile(file, content);
|
||||
}
|
||||
|
||||
async function commit(cwd: string, file: string, message: string) {
|
||||
await write(path.join(cwd, file), `${message}\n`);
|
||||
await git(cwd, "add", file);
|
||||
await git(cwd, "commit", "-q", "-m", message);
|
||||
// Committing leaves COMMIT_EDITMSG, which the storage check rejects on its
|
||||
// own; remove it so these tests reach the history checks.
|
||||
await fs.rm(path.join(cwd, ".git", "COMMIT_EDITMSG"), { force: true });
|
||||
return (await git(cwd, "rev-parse", "HEAD")).trim();
|
||||
}
|
||||
|
||||
describe("assertReproducibleGitFiles with Paperclip-managed files", () => {
|
||||
let root: string;
|
||||
let work: string;
|
||||
let projectSeed: string;
|
||||
let projectRemote: string;
|
||||
let prove: ManagedCheckoutProver;
|
||||
|
||||
// A bare upstream with one commit, plus the seed it was pushed from.
|
||||
async function upstream(name: string, files: Record<string, string>) {
|
||||
const seed = path.join(root, `${name}-seed`);
|
||||
await fs.mkdir(seed, { recursive: true });
|
||||
await git(seed, "init", "-q");
|
||||
for (const [file, content] of Object.entries(files)) await write(path.join(seed, file), content);
|
||||
await git(seed, "add", "-A");
|
||||
await git(seed, "commit", "-q", "-m", "seed");
|
||||
const bare = path.join(root, `${name}.git`);
|
||||
await execFile("git", ["clone", "-q", "--bare", seed, bare]);
|
||||
await git(seed, "remote", "add", "origin", bare);
|
||||
return { seed, bare };
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-reproducible-")));
|
||||
const main = await upstream("main", {
|
||||
"README.md": "main\n",
|
||||
".gitignore": "node_modules/\n.claude/settings.local.json\nsecret.txt\n",
|
||||
});
|
||||
({ seed: projectSeed, bare: projectRemote } = await upstream("project", { "README.md": "project\n", ".gitignore": "node_modules/\n" }));
|
||||
work = path.join(root, "work");
|
||||
await execFile("git", ["clone", "-q", main.bare, work]);
|
||||
// Workspace preparation excludes the managed namespace through info/exclude.
|
||||
await fs.appendFile(path.join(work, ".git", "info", "exclude"), "\n/.paperclip-repositories/\n");
|
||||
prove = (checkout, _name, signal) => assertReproducibleManagedCheckout(checkout, {
|
||||
repoUrl: projectRemote, resolveGitAuth: async () => null, signal,
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function managedCheckout(name = "project-d051a9c4e671") {
|
||||
const checkout = path.join(work, ".paperclip-repositories", name);
|
||||
await fs.mkdir(path.dirname(checkout), { recursive: true });
|
||||
await execFile("git", ["clone", "-q", projectRemote, checkout]);
|
||||
return checkout;
|
||||
}
|
||||
|
||||
const check = () => assertReproducibleGitFiles(work, undefined, { proveManagedCheckout: prove });
|
||||
const expectAccepted = () => expect(check()).resolves.toBeDefined();
|
||||
const expectRejected = (reason: string | RegExp) => expect(check()).rejects.toThrow(reason);
|
||||
|
||||
it("accepts Paperclip's Claude settings file", async () => {
|
||||
await write(path.join(work, ".claude", "settings.local.json"), "{}\n");
|
||||
await expectAccepted();
|
||||
});
|
||||
|
||||
it("accepts a clean managed checkout, including its regenerable directories", async () => {
|
||||
const checkout = await managedCheckout();
|
||||
await write(path.join(checkout, "node_modules", "pkg", "index.js"));
|
||||
await write(path.join(work, "node_modules", "pkg", "index.js"));
|
||||
await expectAccepted();
|
||||
});
|
||||
|
||||
it("rejects managed checkouts unless the caller can prove them", async () => {
|
||||
await managedCheckout();
|
||||
await expect(assertReproducibleGitFiles(work)).rejects.toThrow("unique_ignored_files");
|
||||
});
|
||||
|
||||
it("still rejects unknown ignored files", async () => {
|
||||
await write(path.join(work, "secret.txt"));
|
||||
await expectRejected("unique_ignored_files");
|
||||
});
|
||||
|
||||
it("rejects changed, untracked, and flag-hidden edits", async () => {
|
||||
const checkout = await managedCheckout();
|
||||
await write(path.join(checkout, "README.md"), "edited\n");
|
||||
await expectRejected("dirty_or_untracked_files");
|
||||
await git(checkout, "checkout", "-q", "--", "README.md");
|
||||
await write(path.join(checkout, "notes.txt"));
|
||||
await expectRejected("dirty_or_untracked_files");
|
||||
await fs.rm(path.join(checkout, "notes.txt"));
|
||||
// skip-worktree hides this edit from `git status`.
|
||||
await git(checkout, "update-index", "--skip-worktree", "README.md");
|
||||
await write(path.join(checkout, "README.md"), "hidden edit\n");
|
||||
await expectRejected("unsupported_index_flags");
|
||||
});
|
||||
|
||||
it("rejects commits kept by a branch, a tag, or only the reflog", async () => {
|
||||
const checkout = await managedCheckout();
|
||||
await commit(checkout, "branch.txt", "branch work");
|
||||
await expectRejected("unpushed_managed_checkout");
|
||||
await git(checkout, "tag", "kept");
|
||||
// Loose tag refs fail the storage check on their own; packed ones reach the history check.
|
||||
await git(checkout, "pack-refs", "--all");
|
||||
await git(checkout, "reset", "-q", "--hard", "origin/main");
|
||||
// ORIG_HEAD alone would already reject; drop it to reach the history check.
|
||||
await fs.rm(path.join(checkout, ".git", "ORIG_HEAD"), { force: true });
|
||||
await git(checkout, "reflog", "expire", "--expire=now", "--all");
|
||||
await expectRejected("unpushed_managed_checkout");
|
||||
await git(checkout, "tag", "-d", "kept");
|
||||
await commit(checkout, "amended.txt", "amended work");
|
||||
await git(checkout, "reset", "-q", "--hard", "origin/main");
|
||||
// ORIG_HEAD alone would already reject; drop it to reach the history checks.
|
||||
await fs.rm(path.join(checkout, ".git", "ORIG_HEAD"), { force: true });
|
||||
await expectRejected(/unique_reflog_commits|unpushed_managed_checkout/);
|
||||
});
|
||||
|
||||
it("rejects a detached HEAD commit after its reflog expires", async () => {
|
||||
const checkout = await managedCheckout();
|
||||
await git(checkout, "checkout", "-q", "--detach");
|
||||
await commit(checkout, "detached.txt", "detached work");
|
||||
await git(checkout, "reflog", "expire", "--expire=now", "--all");
|
||||
await expectRejected("unpushed_managed_checkout");
|
||||
});
|
||||
|
||||
it("accepts pushed history after upstream advances", async () => {
|
||||
await managedCheckout();
|
||||
// Upstream moves on after the clone; the checkout never fetches it.
|
||||
await commit(projectSeed, "later.txt", "later upstream work");
|
||||
await git(projectSeed, "push", "-q", "origin", "main");
|
||||
await expectAccepted();
|
||||
});
|
||||
|
||||
it("does not trust a stale remote-tracking ref", async () => {
|
||||
await git(projectSeed, "checkout", "-q", "-b", "feature");
|
||||
await commit(projectSeed, "feature.txt", "feature work");
|
||||
await git(projectSeed, "push", "-q", "origin", "feature");
|
||||
await managedCheckout();
|
||||
// The branch disappears upstream; the checkout's origin/feature is now the last copy.
|
||||
await git(projectRemote, "branch", "-q", "-D", "feature");
|
||||
await expectRejected("unpushed_managed_checkout");
|
||||
});
|
||||
|
||||
it("rejects custom hooks, linked worktrees, stashes, and unreachable objects", async () => {
|
||||
const checkout = await managedCheckout();
|
||||
await write(path.join(checkout, ".git", "hooks", "pre-commit"), "#!/bin/sh\nexit 0\n");
|
||||
await expectRejected("custom_git_metadata");
|
||||
await fs.rm(path.join(checkout, ".git", "hooks", "pre-commit"));
|
||||
|
||||
const linked = path.join(root, "linked");
|
||||
await git(checkout, "worktree", "add", "-q", "--detach", linked);
|
||||
await expectRejected("managed_checkout_has_worktrees");
|
||||
await git(checkout, "worktree", "remove", "--force", linked);
|
||||
|
||||
await write(path.join(checkout, "README.md"), "stashed\n");
|
||||
await git(checkout, "stash", "-q");
|
||||
await expectRejected(/unsupported_local_git_state|unsupported_managed_checkout_refs/);
|
||||
await git(checkout, "stash", "clear");
|
||||
|
||||
await write(path.join(root, "orphan.txt"), "only copy\n");
|
||||
await git(checkout, "hash-object", "-w", path.join(root, "orphan.txt"));
|
||||
await expectRejected("unreachable_managed_checkout_objects");
|
||||
});
|
||||
|
||||
it("never runs a configured fsmonitor command while inspecting", async () => {
|
||||
const checkout = await managedCheckout();
|
||||
const marker = path.join(root, "fsmonitor-ran");
|
||||
const hook = path.join(root, "fsmonitor.sh");
|
||||
await write(hook, `#!/bin/sh\ntouch '${marker}'\nexit 1\n`);
|
||||
await fs.chmod(hook, 0o755);
|
||||
for (const repository of [checkout, work]) {
|
||||
await git(repository, "config", "core.fsmonitor", hook);
|
||||
await check().catch(() => undefined);
|
||||
await git(repository, "config", "--unset", "core.fsmonitor");
|
||||
}
|
||||
await expect(fs.access(marker)).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("never treats task repositories or plain directories as reproducible", async () => {
|
||||
await managedCheckout("task-repo-0123456789abcdef01234567");
|
||||
await expectRejected("unique_ignored_files");
|
||||
await fs.rm(path.join(work, ".paperclip-repositories"), { recursive: true });
|
||||
await write(path.join(work, ".paperclip-repositories", "project-d051a9c4e671", "README.md"));
|
||||
await expectRejected("unproven_managed_checkout");
|
||||
});
|
||||
});
|
||||
@@ -5,8 +5,10 @@ import { executionWorkspaces, issues, heartbeatRuns, nativeRunFinalizations, env
|
||||
workspaceRuntimeServices, issueRecoveryActions, projectWorkspaces, withDedicatedDbConnection, type Db } from "@paperclipai/db";
|
||||
import { createGitRemoteAuthProvider } from "./git-credentials.js";
|
||||
import { withIsolatedRepositorySourceLock, verifiedRecordedIsolatedRemoteSource } from "./isolated-remote-workspace-source.js";
|
||||
import { assertReproducibleGitFiles, assertReproducibleBaseGitStorage, inspectGitCacheRecipe, readGitCacheRestoration,
|
||||
verifyRemoteGitCommit, gitCacheCommand, readGitCacheExpiration, finishGitCacheTrash, type GitCacheExpiration, GIT_CACHE_RESTORATION_KEY, GIT_CACHE_EXPIRATION_REASON } from "./git-workspace-restoration.js";
|
||||
import { managedProjectCheckoutName } from "./managed-project-checkout-name.js";
|
||||
import type { GitRemoteAuthProvider } from "./git-credentials.js";
|
||||
import { assertReproducibleGitFiles, assertReproducibleBaseGitStorage, assertReproducibleManagedCheckout, inspectGitCacheRecipe, readGitCacheRestoration,
|
||||
verifyRemoteGitCommit, type ManagedCheckoutProver, gitCacheCommand, readGitCacheExpiration, finishGitCacheTrash, type GitCacheExpiration, GIT_CACHE_RESTORATION_KEY, GIT_CACHE_EXPIRATION_REASON } from "./git-workspace-restoration.js";
|
||||
|
||||
|
||||
async function allocatedBytes(target: string, shouldContinue: () => boolean): Promise<number> {
|
||||
@@ -122,6 +124,26 @@ async function inspectBase(db: Db, workspace: Workspace, signal?: AbortSignal, a
|
||||
return allocatedBytes(recipe.sourcePath, () => { signal?.throwIfAborted(); return true; });
|
||||
}
|
||||
|
||||
/**
|
||||
* Accepts a managed project checkout only when its name maps back to one of
|
||||
* the workspace's project repositories (workspace preparation re-clones it
|
||||
* from that row on the next run) and the checkout is proven reproducible
|
||||
* against that row's repository URL.
|
||||
*/
|
||||
function managedCheckoutProver(db: Db, workspace: Workspace, auth: GitRemoteAuthProvider): ManagedCheckoutProver {
|
||||
let rows: Promise<Array<{ id: string; repoUrl: string | null; repoRef: string | null; cwd: string | null }>> | undefined;
|
||||
return async (checkoutPath, name, signal) => {
|
||||
if (!workspace.projectId) throw new Error("unknown_managed_checkout");
|
||||
rows ??= db.select({ id: projectWorkspaces.id, repoUrl: projectWorkspaces.repoUrl, repoRef: projectWorkspaces.repoRef,
|
||||
cwd: projectWorkspaces.cwd }).from(projectWorkspaces)
|
||||
.where(and(eq(projectWorkspaces.companyId, workspace.companyId), eq(projectWorkspaces.projectId, workspace.projectId)));
|
||||
const source = (await rows).find(row => row.repoUrl
|
||||
&& managedProjectCheckoutName({ id: row.id, repoUrl: row.repoUrl, repoRef: row.repoRef, cwd: row.cwd }) === name);
|
||||
if (!source?.repoUrl) throw new Error("unknown_managed_checkout");
|
||||
await assertReproducibleManagedCheckout(checkoutPath, { repoUrl: source.repoUrl, resolveGitAuth: auth, signal });
|
||||
};
|
||||
}
|
||||
|
||||
export async function expireReconstructableGitWorkspace(db: Db, snapshot: Workspace, options: {
|
||||
dryRun: boolean; expireBase: boolean; now: Date; signal?: AbortSignal;
|
||||
beforeDelete?: (workspace: Workspace) => Promise<void>;
|
||||
@@ -141,8 +163,10 @@ export async function expireReconstructableGitWorkspace(db: Db, snapshot: Worksp
|
||||
// A restored active workspace starts a new expiration cycle and records its new HEAD.
|
||||
if (!hasPendingGitExpiration(workspace) && !["active", "idle", "in_review"].includes(workspace.status)) throw new Error("not_expirable");
|
||||
if (["active", "idle", "in_review"].includes(workspace.status)) state = null;
|
||||
const auth = createGitRemoteAuthProvider(db, workspace.companyId, { issueId: workspace.sourceIssueId! });
|
||||
const proveManagedCheckout = managedCheckoutProver(db, workspace, auth);
|
||||
let recipe = state ? readGitCacheRestoration(workspace)! : await inspectGitCacheRecipe(workspace,
|
||||
createGitRemoteAuthProvider(db, workspace.companyId, { issueId: workspace.sourceIssueId! }), options.signal);
|
||||
auth, options.signal, { proveManagedCheckout });
|
||||
const present = await exists(recipe.worktreePath);
|
||||
if (!state && !present) throw new Error("missing_unverified_worktree");
|
||||
result.candidate = true;
|
||||
@@ -185,7 +209,7 @@ export async function expireReconstructableGitWorkspace(db: Db, snapshot: Worksp
|
||||
const { acquireGitWorktreeCleanupLock } = await import("./workspace-runtime.js");
|
||||
const lock = await acquireGitWorktreeCleanupLock(recipe.worktreePath);
|
||||
try {
|
||||
await assertReproducibleGitFiles(recipe.worktreePath, options.signal);
|
||||
await assertReproducibleGitFiles(recipe.worktreePath, options.signal, { proveManagedCheckout });
|
||||
if ((await gitCacheCommand(recipe.worktreePath, ["rev-parse", "HEAD"], options.signal)).trim() !== recipe.commit) throw new Error("head_changed");
|
||||
await fs.rename(recipe.worktreePath, state.worktreeTrash);
|
||||
await finishGitCacheTrash(workspace);
|
||||
|
||||
@@ -10,6 +10,7 @@ import type { GitRemoteAuthProvider } from "./git-credentials.js";
|
||||
import { isRuntimeOwnedGitBranch } from "./execution-workspace-branch-ownership.js";
|
||||
import { recordedIsolatedRemoteSource, verifiedRecordedIsolatedRemoteSource, materializeIsolatedRemoteWorkspaceSource } from "./isolated-remote-workspace-source.js";
|
||||
import { findRegenerableWorkspaceDirs } from "./workspace-compaction.js";
|
||||
import { PROJECT_REPOSITORIES_DIR } from "@paperclipai/adapter-utils/git-workspace-sync";
|
||||
|
||||
const execFile = promisify(execFileCallback);
|
||||
const shaPattern = /^[a-f0-9]{40}$/;
|
||||
@@ -92,7 +93,10 @@ export function readGitCacheRestoration(workspace: Workspace): GitCacheRestorati
|
||||
}
|
||||
|
||||
export async function gitCacheCommand(cwd: string, args: string[], signal?: AbortSignal) {
|
||||
const result = await execFile("git", ["-C", cwd, ...args], {
|
||||
// The inspected repositories are agent-writable. core.fsmonitor in their
|
||||
// local config would make `git status` run an arbitrary command as the
|
||||
// server; the command-line setting takes precedence over it.
|
||||
const result = await execFile("git", ["-c", "core.fsmonitor=false", "-C", cwd, ...args], {
|
||||
timeout: 60_000, signal, maxBuffer: 16 * 1024 * 1024,
|
||||
env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_OPTIONAL_LOCKS: "0" },
|
||||
});
|
||||
@@ -170,15 +174,128 @@ export async function assertReproducibleBaseGitStorage(cwd: string, signal?: Abo
|
||||
for (const directory of ["objects", "refs", "logs"]) await walk(directory);
|
||||
}
|
||||
|
||||
/** Inspect every untracked/ignored file. Only repository-ignored regenerable directories qualify. */
|
||||
export async function assertReproducibleGitFiles(cwd: string, signal?: AbortSignal) {
|
||||
/**
|
||||
* Paperclip writes this file into the worktree on every Claude run (it merges
|
||||
* its permission rules into whatever is there), so a terminal worktree's copy
|
||||
* is regenerated when the task runs again.
|
||||
*/
|
||||
const PAPERCLIP_CLAUDE_SETTINGS_FILE = ".claude/settings.local.json";
|
||||
|
||||
/**
|
||||
* Proves that a managed project checkout (`.paperclip-repositories/<name>`)
|
||||
* holds nothing that re-cloning would not restore; throws otherwise. Only the
|
||||
* expiration worker supplies one, because it can map the checkout back to its
|
||||
* project repository and reach upstream.
|
||||
*/
|
||||
export type ManagedCheckoutProver = (checkoutPath: string, name: string, signal?: AbortSignal) => Promise<void>;
|
||||
|
||||
/**
|
||||
* The proof for one managed checkout, against the repository URL recorded on
|
||||
* its project workspace row (never the checkout's own, agent-writable config):
|
||||
* the checkout's tree, index, config and Git storage hold nothing unique, it
|
||||
* has no linked worktrees, and every commit reachable from its refs or
|
||||
* reflogs is reachable from a current upstream branch or tag.
|
||||
*/
|
||||
export async function assertReproducibleManagedCheckout(checkout: string, input: {
|
||||
repoUrl: string; resolveGitAuth: GitRemoteAuthProvider; signal?: AbortSignal;
|
||||
}) {
|
||||
const { signal } = input;
|
||||
const gitDir = await fs.lstat(path.join(checkout, ".git")).catch(() => null);
|
||||
if (!gitDir?.isDirectory()) throw new Error("unproven_managed_checkout");
|
||||
if ((await gitCacheCommand(checkout, ["rev-parse", "--show-toplevel"], signal)).trim() !== await fs.realpath(checkout)) {
|
||||
throw new Error("unproven_managed_checkout");
|
||||
}
|
||||
const worktrees = (await gitCacheCommand(checkout, ["worktree", "list", "--porcelain"], signal))
|
||||
.split("\n").filter(line => line.startsWith("worktree "));
|
||||
if (worktrees.length !== 1) throw new Error("managed_checkout_has_worktrees");
|
||||
// Tree, index flags, local config, ignored files and in-progress state, as
|
||||
// for any worktree; nested managed checkouts are never accepted here.
|
||||
await assertReproducibleGitFiles(checkout, signal);
|
||||
// Hooks, info files and object/ref storage must be stock.
|
||||
await assertReproducibleBaseGitStorage(checkout, signal);
|
||||
const refs = (await gitCacheCommand(checkout, ["for-each-ref", "--format=%(refname)"], signal)).trim().split("\n").filter(Boolean);
|
||||
if (refs.some(ref => !/^refs\/(heads|tags|remotes\/origin)\//.test(ref))) throw new Error("unsupported_managed_checkout_refs");
|
||||
// Every commit a ref or reflog keeps must be on a current upstream branch or
|
||||
// tag. Local remote-tracking refs prove nothing on their own (they may be
|
||||
// stale), so the proof fetches upstream's refs fresh.
|
||||
// HEAD is listed explicitly: a detached HEAD is not a named ref, and once
|
||||
// its reflog expires nothing else would name its commits.
|
||||
const localTips = new Set([
|
||||
...(await gitCacheCommand(checkout, ["rev-parse", "--verify", "HEAD^{commit}"], signal)).split("\n"),
|
||||
...(await gitCacheCommand(checkout, ["for-each-ref", "--format=%(objectname)"], signal)).split("\n"),
|
||||
...(await gitCacheCommand(checkout, ["reflog", "show", "--all", "--format=%H"], signal).catch(() => "")).split("\n"),
|
||||
].map(line => line.trim()).filter(sha => shaPattern.test(sha)));
|
||||
if (localTips.size > MANAGED_CHECKOUT_LOCAL_TIP_LIMIT) throw new Error("managed_checkout_local_tip_budget");
|
||||
if (localTips.size && await hasCommitsMissingUpstream(checkout, [...localTips], input.repoUrl, input.resolveGitAuth, signal)) {
|
||||
throw new Error("unpushed_managed_checkout");
|
||||
}
|
||||
// Objects no ref or reflog reaches (a reset `git add`, a dropped stash) may
|
||||
// be the only copy of some work.
|
||||
const unreachable = await gitCacheCommand(checkout, ["fsck", "--connectivity-only", "--unreachable", "--no-progress"], signal);
|
||||
if (unreachable.trim()) throw new Error("unreachable_managed_checkout_objects");
|
||||
}
|
||||
|
||||
/** Upper bound on distinct ref and reflog commits a proof will consider. */
|
||||
const MANAGED_CHECKOUT_LOCAL_TIP_LIMIT = 5_000;
|
||||
|
||||
/**
|
||||
* Fetches upstream's current branches and tags into a throwaway proof store
|
||||
* that borrows the checkout's objects (so only commits the checkout lacks are
|
||||
* downloaded), then asks whether any local commit is unreachable from them.
|
||||
* It is one fetch and one rev-list however many refs either side has. The
|
||||
* fetch runs config-free with the company's Git identity, never the
|
||||
* checkout's own config.
|
||||
*/
|
||||
async function hasCommitsMissingUpstream(checkout: string, localTips: string[], repoUrl: string,
|
||||
resolveGitAuth: GitRemoteAuthProvider, signal?: AbortSignal) {
|
||||
const store = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-git-checkout-proof-"));
|
||||
try {
|
||||
const auth = await resolveGitAuth(repoUrl);
|
||||
const env = { ...process.env, ...auth?.env, GIT_TERMINAL_PROMPT: "0", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: os.devNull };
|
||||
const options = { timeout: 120_000, signal, maxBuffer: 16 * 1024 * 1024, env, cwd: os.tmpdir() };
|
||||
await execFile("git", ["init", "--bare", "--quiet", store], options);
|
||||
await fs.writeFile(path.join(store, "objects", "info", "alternates"), `${path.join(await fs.realpath(checkout), ".git", "objects")}\n`);
|
||||
await execFile("git", [...auth?.configArgs ?? [], "-C", store, "fetch", "--quiet", "--no-tags", "--no-recurse-submodules",
|
||||
"--", repoUrl, "+refs/heads/*:refs/upstream/heads/*", "+refs/tags/*:refs/upstream/tags/*"], options);
|
||||
const missing = await execFile("git", ["-C", store, "rev-list", "--max-count=1", ...localTips, "--not", "--glob=refs/upstream/*"], options);
|
||||
return missing.stdout.trim() !== "";
|
||||
} finally {
|
||||
await fs.rm(store, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Inspect every untracked/ignored file. Only repository-ignored regenerable
|
||||
* directories, Paperclip's Claude settings file, and managed project checkouts
|
||||
* that `proveManagedCheckout` accepts qualify.
|
||||
*/
|
||||
export async function assertReproducibleGitFiles(cwd: string, signal?: AbortSignal, options: {
|
||||
proveManagedCheckout?: ManagedCheckoutProver;
|
||||
} = {}) {
|
||||
if ((await gitCacheCommand(cwd, ["status", "--porcelain=v1", "--untracked-files=all", "-z"], signal)).length) throw new Error("dirty_or_untracked_files");
|
||||
if ((await gitCacheCommand(cwd, ["ls-files", "-v"], signal)).split("\n").some(line => /^[a-zS] /.test(line))) throw new Error("unsupported_index_flags");
|
||||
const localConfig = await gitCacheCommand(cwd, ["config", "--local", "--name-only", "--list"], signal);
|
||||
if (localConfig.split("\n").filter(Boolean).some(key => !/^(core\.(repositoryformatversion|filemode|bare|logallrefupdates|ignorecase|precomposeunicode)|remote\.origin\.(url|fetch)|branch\.[^.]+\.(remote|merge))$/.test(key))) throw new Error("unsupported_local_git_config");
|
||||
const regenerable = await findRegenerableWorkspaceDirs(cwd, { signal });
|
||||
const ignored = (await gitCacheCommand(cwd, ["ls-files", "--others", "--ignored", "--exclude-standard", "-z"], signal)).split("\0").filter(Boolean);
|
||||
if (ignored.some(file => !regenerable.some(dir => file.startsWith(`${dir}/`)))) throw new Error("unique_ignored_files");
|
||||
const unexplained = ignored.filter(file => file !== PAPERCLIP_CLAUDE_SETTINGS_FILE && !regenerable.some(dir => file.startsWith(`${dir}/`)));
|
||||
// Anything else unexplained must sit inside a managed project checkout that
|
||||
// the caller can prove reproducible. Task repositories (`task-repo-*`) carry
|
||||
// access receipts and staging clones (`*.clone-*`) are mid-publication, so
|
||||
// neither ever qualifies.
|
||||
const checkouts = new Set<string>();
|
||||
for (const file of unexplained) {
|
||||
const [namespace, name, rest] = file.split("/", 3);
|
||||
if (namespace !== PROJECT_REPOSITORIES_DIR || !name || rest === undefined || !options.proveManagedCheckout
|
||||
|| name.startsWith("task-repo-") || name.includes(".clone-")) throw new Error("unique_ignored_files");
|
||||
checkouts.add(name);
|
||||
}
|
||||
for (const name of checkouts) {
|
||||
const checkout = path.join(cwd, PROJECT_REPOSITORIES_DIR, name);
|
||||
const entry = await fs.lstat(checkout);
|
||||
if (!entry.isDirectory() || entry.isSymbolicLink()) throw new Error("unique_ignored_files");
|
||||
await options.proveManagedCheckout!(checkout, name, signal);
|
||||
}
|
||||
const entries = await gitCacheCommand(cwd, ["ls-tree", "-r", "HEAD"], signal);
|
||||
if (/^160000 /m.test(entries)) throw new Error("unsupported_submodules");
|
||||
const gitDir = (await gitCacheCommand(cwd, ["rev-parse", "--absolute-git-dir"], signal)).trim();
|
||||
@@ -222,7 +339,8 @@ export async function assertReproducibleGitFiles(cwd: string, signal?: AbortSign
|
||||
return regenerable;
|
||||
}
|
||||
|
||||
export async function inspectGitCacheRecipe(workspace: Workspace, resolveGitAuth: GitRemoteAuthProvider, signal?: AbortSignal): Promise<GitCacheRestoration> {
|
||||
export async function inspectGitCacheRecipe(workspace: Workspace, resolveGitAuth: GitRemoteAuthProvider, signal?: AbortSignal,
|
||||
options: { proveManagedCheckout?: ManagedCheckoutProver } = {}): Promise<GitCacheRestoration> {
|
||||
if (workspace.mode !== "isolated_workspace" || workspace.strategyType !== "git_worktree"
|
||||
|| !isRuntimeOwnedGitBranch(workspace.metadata) || !workspace.sourceIssueId || !workspace.repoUrl
|
||||
|| !workspace.cwd || !workspace.branchName || (workspace.providerRef && workspace.providerRef !== workspace.cwd)) throw new Error("unproven_workspace_ownership");
|
||||
@@ -241,7 +359,7 @@ export async function inspectGitCacheRecipe(workspace: Workspace, resolveGitAuth
|
||||
if (!(await fs.lstat(path.join(workspace.cwd, ".git"))).isFile()) throw new Error("not_linked_worktree");
|
||||
if ((await gitCacheCommand(workspace.cwd, ["rev-parse", "--path-format=absolute", "--git-common-dir"], signal)).trim() !== path.join(await fs.realpath(source), ".git")) throw new Error("source_identity_changed");
|
||||
if ((await gitCacheCommand(workspace.cwd, ["symbolic-ref", "--short", "HEAD"], signal)).trim() !== workspace.branchName) throw new Error("branch_identity_changed");
|
||||
await assertReproducibleGitFiles(workspace.cwd, signal);
|
||||
await assertReproducibleGitFiles(workspace.cwd, signal, options);
|
||||
const commit = (await gitCacheCommand(workspace.cwd, ["rev-parse", "HEAD"], signal)).trim();
|
||||
await verifyRemoteGitCommit(workspace.repoUrl, commit, resolveGitAuth, signal);
|
||||
return { schema: "paperclip.git-cache-restoration.v1", companyId: workspace.companyId, workspaceId: workspace.id,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { assertIsolatedWorkspaceSourceReadAccess, retainIsolatedWorkspaceSourceAccess } from "../execution-workspace-repositories.js";
|
||||
import { deriveRepoNameFromRepoUrl, managedProjectCheckoutName, projectRepositoryIdentity } from "../managed-project-checkout-name.js";
|
||||
import { forbidden } from "../../errors.js";
|
||||
import { readGitConnectionFailure } from "../git-connection-failure.js";
|
||||
import { materializeManagedProjectWorkspace, ensureManagedRepositoriesIgnored } from "../managed-repository-checkout.js";
|
||||
@@ -423,24 +424,6 @@ export async function resolveWorkspaceAfterLowTrustPreflight<
|
||||
};
|
||||
}
|
||||
|
||||
function deriveRepoNameFromRepoUrl(repoUrl: string | null): string | null {
|
||||
const trimmed = repoUrl?.trim() ?? "";
|
||||
if (!trimmed) return null;
|
||||
try {
|
||||
const parsed = new URL(trimmed);
|
||||
const cleanedPath = parsed.pathname.replace(/\/+$/, "");
|
||||
const repoName =
|
||||
cleanedPath
|
||||
.split("/")
|
||||
.filter(Boolean)
|
||||
.pop()
|
||||
?.replace(/\.git$/i, "") ?? "";
|
||||
return repoName || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
type PrepareManagedLocalSource = (sourceCwd: string, targetCwd: string) => Promise<(targetCwd: string) => Promise<void>>;
|
||||
|
||||
/**
|
||||
@@ -516,7 +499,7 @@ export async function prepareProjectRepositoryWorkspaces(input: {
|
||||
beforePublish?: () => Promise<void>;
|
||||
prepareLocalSource?: (sourceCwd: string) => Promise<() => Promise<void>>;
|
||||
}): Promise<Array<{ workspaceId: string; cwd: string; repoUrl: string; repoRef: string | null }>> {
|
||||
const identity = (url: string) => url.trim().replace(/\.git\/?$/, "").replace(/\/$/, "");
|
||||
const identity = projectRepositoryIdentity;
|
||||
const seen = new Set(input.anchorRepoUrl ? [identity(input.anchorRepoUrl)] : []);
|
||||
const selected = input.workspaces.filter((workspace) => {
|
||||
if (!workspace.repoUrl || seen.has(identity(workspace.repoUrl))) return false;
|
||||
@@ -533,8 +516,7 @@ export async function prepareProjectRepositoryWorkspaces(input: {
|
||||
const results = [];
|
||||
for (const workspace of selected) {
|
||||
const repoUrl = workspace.repoUrl!;
|
||||
const name = (deriveRepoNameFromRepoUrl(repoUrl) ?? "repo").replace(/[^a-zA-Z0-9_-]/g, "-");
|
||||
const key = `${name}-${createHash("sha256").update(JSON.stringify([workspace.id, identity(repoUrl), workspace.repoRef, workspace.cwd ?? null])).digest("hex").slice(0, 12)}`;
|
||||
const key = managedProjectCheckoutName({ id: workspace.id, repoUrl, repoRef: workspace.repoRef, cwd: workspace.cwd });
|
||||
const cwd = path.join(root, key);
|
||||
const existing = await fs.lstat(cwd).catch(() => null);
|
||||
if (existing && (!existing.isDirectory() || existing.isSymbolicLink())) throw new Error("Invalid project repository checkout path");
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
/** Normalizes a repository URL so `.git` and trailing-slash variants match. */
|
||||
export function projectRepositoryIdentity(url: string) {
|
||||
return url.trim().replace(/\.git\/?$/, "").replace(/\/$/, "");
|
||||
}
|
||||
|
||||
export function deriveRepoNameFromRepoUrl(repoUrl: string | null): string | null {
|
||||
const trimmed = repoUrl?.trim() ?? "";
|
||||
if (!trimmed) return null;
|
||||
try {
|
||||
const parsed = new URL(trimmed);
|
||||
const cleanedPath = parsed.pathname.replace(/\/+$/, "");
|
||||
const repoName =
|
||||
cleanedPath
|
||||
.split("/")
|
||||
.filter(Boolean)
|
||||
.pop()
|
||||
?.replace(/\.git$/i, "") ?? "";
|
||||
return repoName || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Directory name of a project repository's managed checkout under
|
||||
* `.paperclip-repositories/`. It is derived from the project workspace row,
|
||||
* so the same row always maps to the same checkout, and a checkout name maps
|
||||
* back to the row it was cloned from.
|
||||
*/
|
||||
export function managedProjectCheckoutName(workspace: {
|
||||
id: string; repoUrl: string; repoRef: string | null; cwd?: string | null;
|
||||
}) {
|
||||
const name = (deriveRepoNameFromRepoUrl(workspace.repoUrl) ?? "repo").replace(/[^a-zA-Z0-9_-]/g, "-");
|
||||
const hash = createHash("sha256")
|
||||
.update(JSON.stringify([workspace.id, projectRepositoryIdentity(workspace.repoUrl), workspace.repoRef, workspace.cwd ?? null]))
|
||||
.digest("hex").slice(0, 12);
|
||||
return `${name}-${hash}`;
|
||||
}
|
||||
Reference in new issue
Block a user